Packages and utilities

# Provides bootstrap resampling tools
library(rsample)
Error in library(rsample) : there is no package called ‘rsample’
# Compute the log-likelihood of a new dataset using a fit lme4 model.
logLik_test <- function(lm, test_X, test_y) {
  predictions <- predict(lm, test_X, re.form=NA)
  # Get std.dev. of residual, estimated from train data
  stdev <- sigma(lm)
  # For each prediction--observation, get the density p(obs | N(predicted, model_sigma)) and reduce
  density <- sum(dnorm(test_y, predictions, stdev, log=TRUE))
  return(density)
}
# Get per-prediction log-likelihood
logLik_test_per <- function(lm, test_X, test_y) {
  predictions <- predict(lm, test_X, re.form=NA)
  # Get std.dev. of residual, estimated from train data
  stdev <- sigma(lm)
  # For each prediction--observation, get the density p(obs | N(predicted, model_sigma))
  densities <- dnorm(test_y, predictions, stdev, log=TRUE)
  return(densities)
}
# Compute MSE of a new dataset using a fit lme4 model.
mse_test <- function(lm, test_X, test_y) {
  return(mean((predict(lm, test_X, re.form=NA) - test_y) ^ 2))
}
#Sanity checks
#mylm <- gam(psychometric ~  s(surprisal, bs = "cr", k = 20) + s(prev_surp, bs = "cr", k = 20) + te(freq, len, bs = "cr") + te(prev_freq, prev_len, bs = "cr"), data=train_data)
#c(logLik(mylm), logLik_test(mylm, train_data, train_data$psychometric))
#logLik_test(mylm, test_data, test_data$psychometric)

Data loading and preprocessing

data = read.csv("../data/harmonized_results.csv")

all_data = data %>%
  mutate(seed = as.factor(seed)) %>%
  group_by(corpus, model, training, seed) %>%
    mutate(prev_surp = lag(surprisal),
         prev_code = lag(code),
         prev_len = lag(len),
         prev_freq = lag(freq),
         prev_surp = lag(surprisal),
         
         prev2_freq = lag(prev_freq),
         prev2_code = lag(prev_code),
         prev2_len = lag(prev_len),
         prev2_surp = lag(prev_surp),
         
         prev3_freq = lag(prev2_freq),
         prev3_code = lag(prev2_code),
         prev3_len = lag(prev2_len),
         prev3_surp = lag(prev2_surp),
         
         prev4_freq = lag(prev3_freq),
         prev4_code = lag(prev3_code),
         prev4_len = lag(prev3_len),
         prev4_surp = lag(prev3_surp)) %>%
  ungroup() %>%
  
  # Filter back two for the dundee corpus. Filter back 1 for all other corpora
  # NB this effectively removes all zero-surprisal rows, since early-sentence tokens don't have contiguous token history
  filter((corpus == "dundee" & code == prev2_code + 2) | (corpus != "dundee" & code == prev4_code + 4)) %>%
  
  select(-prev_code, -prev2_code, -prev3_code) %>%
  drop_na()

all_data = all_data %>%
  mutate(
    model = as.character(model),
    model = if_else(model == "gpt-2", "gpt2", model),
    model = as.factor(model))
missing_rows = all_data %>% complete(nesting(corpus, code), nesting(model, training, seed)) %>% 
  group_by(corpus, code) %>% 
    filter(sum(is.na(surprisal)) > 0) %>% 
  ungroup() %>% 
  anti_join(all_data, by=c("corpus", "code", "model", "training", "seed"))

missing_rows %>% ggplot(aes(x=corpus, fill=factor(paste(model,training)))) + geom_bar(position=position_dodge(width=0.8))

print(missing_rows %>% group_by(model, training, seed, corpus) %>% summarise(n=n())) %>% arrange(desc(n))

# Compute the ideal number of model--seed--training observations per token.
to_drop = all_data %>%
  group_by(corpus, code) %>% summarise(n = n()) %>% ungroup() %>%
  group_by(corpus) %>% mutate( max_n = max(n)) %>% ungroup() %>%
  filter(max_n != n) %>%
  select(code, corpus)

#to_drop = all_data %>% group_by(corpus, code) %>% filter(n() != ideal_token_obs_count) %>% ungroup()
loginfo(paste("Dropping", nrow(to_drop), "observations corresponding to corpus tokens which are missing observations for some model."))
2020-05-29 10:42:54 INFO::Dropping 10342 observations corresponding to corpus tokens which are missing observations for some model.
loginfo(paste("Dropping", to_drop %>% group_by(corpus, code) %>% n_groups(), "tokens which are missing observations for some model."))
2020-05-29 10:42:54 INFO::Dropping 10342 tokens which are missing observations for some model.
all_data = all_data %>% anti_join(to_drop %>% group_by(corpus, code), by=c("corpus", "code"))
loginfo(paste("After drop,", nrow(all_data), "observations (", all_data %>% group_by(corpus, code) %>% n_groups(), " tokens) remain."))
2020-05-29 10:42:55 INFO::After drop, 962274 observations ( 33117  tokens) remain.

to_drop_zero_surps = all_data %>% group_by(corpus, code) %>% filter(any(surprisal == 0)) %>% ungroup()
loginfo(paste("Dropping", nrow(to_drop_zero_surps), "observations corresponding to corpus tokens which have surprisal zeros for some model."))
2020-05-29 10:42:56 INFO::Dropping 116 observations corresponding to corpus tokens which have surprisal zeros for some model.
loginfo(paste("Dropping", to_drop_zero_surps %>% group_by(corpus, code) %>% n_groups(), "tokens which have surprisal zeros for some model."))
2020-05-29 10:42:56 INFO::Dropping 4 tokens which have surprisal zeros for some model.
all_data = all_data %>% anti_join(to_drop_zero_surps %>% group_by(corpus, code), by=c("corpus", "code"))
loginfo(paste("After drop,", nrow(all_data), "observations (", all_data %>% group_by(corpus, code) %>% n_groups(), " tokens) remain."))
2020-05-29 10:42:56 INFO::After drop, 962158 observations ( 33113  tokens) remain.

to_drop_zero_psychs = all_data %>% group_by(corpus, code) %>% filter(any(psychometric == 0)) %>% ungroup()
loginfo(paste("Dropping", nrow(to_drop_zero_psychs), "observations corresponding to corpus tokens which have psychometric zeros for some model."))
2020-05-29 10:42:57 INFO::Dropping 14935 observations corresponding to corpus tokens which have psychometric zeros for some model.
loginfo(paste("Dropping", to_drop_zero_psychs %>% group_by(corpus, code) %>% n_groups(), "tokens which have psychometric zeros for some model."))
2020-05-29 10:42:57 INFO::Dropping 515 tokens which have psychometric zeros for some model.
all_data = all_data %>% anti_join(to_drop_zero_psychs %>% group_by(corpus, code), by=c("corpus", "code"))
loginfo(paste("After drop,", nrow(all_data), "observations (", all_data %>% group_by(corpus, code) %>% n_groups(), " tokens) remain."))
2020-05-29 10:42:57 INFO::After drop, 947223 observations ( 32598  tokens) remain.

Learn models

# Compute linear model stats for the given training data subset and full test data.
# Automatically subsets the test data to match the relevant group for which we are training a linear model.
get_lm_data <- function(df, test_data, formula, fold, store_env) {
  #this_lm <- gam(formula, data=df);
  this_lm = lm(formula, data=df)
  this_test_data <- semi_join(test_data, df, by=c("training", "model", "seed", "corpus"));
  
  # Save lm to the global env so that we can access residuals later.
  lm_name = paste(unique(paste(df$model, df$training, df$seed, df$corpus))[1], fold)
  assign(lm_name, this_lm, envir=store_env)
  
  summarise(df,
            log_lik = as.numeric(logLik(this_lm, REML = F)),
            test_lik = logLik_test(this_lm, this_test_data, this_test_data$psychometric),
            test_mse = mse_test(this_lm, this_test_data, this_test_data$psychometric))
}
# For a previously fitted lm stored in store_env, get the residuals on test data of the relevant data subset.
get_lm_residuals <- function(df, fold, store_env) {
  # Retrieve the relevant lm.
  lm_name = paste(unique(paste(df$model, df$training, df$seed, df$corpus))[1], fold)
  this_lm <- get(lm_name, envir=store_env)
  
  mutate(df,
         likelihood = logLik_test_per(this_lm, df, df$psychometric),
         resid = df$psychometric - predict(this_lm, df, re.form=NA))
}
# Compute per-example delta-log-likelihood for the given test fold.
get_lm_delta_log_lik <- function(test_data, fold, baseline_env, full_env) {
  lm_name = paste(unique(paste(test_data$model, test_data$training, test_data$seed, test_data$corpus))[1], fold)
  baseline_lm <- get(lm_name, envir=baseline_env)
  full_lm <- get(lm_name, envir=full_env)
  
  delta_log_lik = logLik_test_per(full_lm, test_data, test_data$psychometric) - logLik_test_per(baseline_lm, test_data, test_data$psychometric)
  return(cbind(test_data, delta_log_lik=delta_log_lik))
}
#####
# Define regression formulae.
# Eye-tracking regression: only use surprisal and previous surprisal; SPRT regression: use 2-back features.

#baseline_rt_regression = psychometric ~ te(freq, len, bs = "cr") + te(prev_freq, prev_len, bs = "cr") + te(prev2_freq, prev2_len, bs = "cr")
#baselie_sprt_regression = psychometric ~ te(freq, len, bs = "cr") + te(prev_freq, prev_len, bs = "cr") + te(prev2_freq, prev2_len, bs = "cr") + te(prev3_freq, prev3_len, bs = "cr") + te(prev4_freq, prev4_len, bs = "cr")

#full_rt_regression = psychometric ~ s(surprisal, bs = "cr", k = 20) + s(prev_surp, bs = "cr", k = 20) + s(prev2_surp, bs = "cr", k = 20) + te(freq, len, bs = "cr") + te(prev_freq, prev_len, bs = "cr") + te(prev2_freq, prev2_len, bs = "cr")
#full_sprt_regression = psychometric ~ s(surprisal, bs = "cr", k = 20) + s(prev_surp, bs = "cr", k = 20) + s(prev2_surp, bs = "cr", k = 20) + s(prev3_surp, bs = "cr", k = 20) + s(prev4_surp, bs = "cr", k = 20) + te(freq, len, bs = "cr") + te(prev_freq, prev_len, bs = "cr") + te(prev2_freq, prev2_len, bs = "cr") + te(prev3_freq, prev3_len, bs = "cr") + te(prev4_freq, prev4_len, bs = "cr")

baseline_rt_regression = psychometric ~ freq + prev_freq + prev2_freq + len + prev_len + prev2_len
baseline_sprt_regression = psychometric ~ freq + prev_freq + prev2_freq + prev3_freq + prev4_freq + len + prev_len + prev2_len + prev3_len + prev4_len

full_sprt_regression = psychometric ~ surprisal + prev_surp + prev2_surp + prev3_surp + prev4_surp + freq + prev_freq + prev2_freq + prev3_freq + prev4_freq + len + prev_len + prev2_len + prev3_len + prev4_len
full_rt_regression = psychometric ~ surprisal + prev_surp + prev2_surp + freq + prev_freq + prev2_freq + len + prev_len + prev2_len
  
#####
# Prepare frames/environments for storing results/objects.
baseline_results = data.frame()
full_model_results = data.frame()
baseline_residuals = data.frame()
full_residuals = data.frame()
log_lik_deltas = data.frame()

#Randomly shuffle the data
all_data<-all_data[sample(nrow(all_data)),]
#Create K equally size folds
K = 10
folds <- cut(seq(1,nrow(all_data)),breaks=K,labels=FALSE)
#Perform 10 fold cross validation

# Fit models for some fold of the data.
baseline_corpus = function(corpus, df, test_data, fold, env) {
  if(corpus == "dundee") {
    get_lm_data(df, test_data, baseline_rt_regression, fold, env)
  } else {
    get_lm_data(df, test_data, baseline_sprt_regression, fold, env)
  }
}
full_model_corpus = function(corpus, df, test_data, fold, env) {
  if(corpus[1] == "dundee") {
    get_lm_data(df, test_data, full_rt_regression, fold, env)
  } else {
    get_lm_data(df, test_data, full_sprt_regression, fold, env)
  }
}

# Prepare a new Environment in which we store fitted LMs, which we'll query later for residuals and other metrics.
baseline_env = new.env()
full_env = new.env()

for(i in 1:K) { 
  #Segement your data by fold using the which() function 
  testIndexes <- which(folds==i, arr.ind=TRUE)
  test_data <- all_data[testIndexes, ]
  train_data <- all_data[-testIndexes, ]
  
  # Compute a baseline linear model for each model--training--seed--RT-corpus combination.
  baselines = train_data %>%
    group_by(model, training, seed, corpus) %>%
      print(model) %>%
      do(baseline_corpus(unique(.$corpus), ., test_data, i, baseline_env)) %>%
    ungroup() %>%
    mutate(seed = as.factor(seed),
           fold = i)
  
  baseline_results = rbind(baseline_results, baselines)
  
  # Compute a full linear model for each model--training--seed-RT-corpus combination
  full_models = train_data %>%
    group_by(model, training, seed, corpus) %>%
      do(full_model_corpus(unique(.$corpus), ., test_data, i, full_env)) %>%
    ungroup() %>%
    mutate(seed = as.factor(seed),
           fold = i)
  
  full_model_results = rbind(full_model_results, full_models)
  
  # Compute delta-log-likelihoods
  fold_log_lik_deltas = test_data %>%
    group_by(model, training, seed, corpus) %>%
      do(get_lm_delta_log_lik(., i, baseline_env, full_env)) %>%
    ungroup()

  log_lik_deltas = rbind(log_lik_deltas, fold_log_lik_deltas)
  
  fold_baseline_residuals = test_data %>%
    group_by(model, training, seed, corpus) %>%
      do(get_lm_residuals(., i, baseline_env)) %>%
    ungroup()

  baseline_residuals = rbind(baseline_residuals, fold_baseline_residuals)

  fold_full_residuals = test_data %>%
    group_by(model, training, seed, corpus) %>%
      do(get_lm_residuals(., i, full_env)) %>%
    ungroup()

  full_residuals = rbind(full_residuals, fold_full_residuals)
}

|====================================================================================================                                                                                 | 56% ~2 s remaining     
|======================================================================================================                                                                               | 57% ~2 s remaining     
|=============================================================================================================                                                                        | 60% ~1 s remaining     
|===================================================================================================================                                                                  | 64% ~1 s remaining     
|=========================================================================================================================                                                            | 67% ~1 s remaining     
|===============================================================================================================================                                                      | 70% ~1 s remaining     
|=====================================================================================================================================                                                | 74% ~1 s remaining     
|===========================================================================================================================================                                          | 77% ~1 s remaining     
|=============================================================================================================================================                                        | 78% ~1 s remaining     
|====================================================================================================================================================                                 | 82% ~1 s remaining     
|==========================================================================================================================================================                           | 85% ~1 s remaining     
|================================================================================================================================================================                     | 89% ~0 s remaining     
|====================================================================================================================================================================                 | 91% ~0 s remaining     
|======================================================================================================================================================================               | 92% ~0 s remaining     
|============================================================================================================================================================================         | 95% ~0 s remaining     
|==================================================================================================================================================================================   | 99% ~0 s remaining     
|======================================================================================================                                                                               | 57% ~2 s remaining     
|==========================================================================================================                                                                           | 59% ~1 s remaining     
|=============================================================================================================                                                                        | 60% ~1 s remaining     
|=================================================================================================================                                                                    | 62% ~1 s remaining     
|===================================================================================================================                                                                  | 64% ~1 s remaining     
|=======================================================================================================================                                                              | 66% ~1 s remaining     
|=========================================================================================================================                                                            | 67% ~1 s remaining     
|=============================================================================================================================                                                        | 69% ~1 s remaining     
|===============================================================================================================================                                                      | 70% ~1 s remaining     
|===================================================================================================================================                                                  | 73% ~1 s remaining     
|=====================================================================================================================================                                                | 74% ~1 s remaining     
|=========================================================================================================================================                                            | 76% ~1 s remaining     
|=============================================================================================================================================                                        | 78% ~1 s remaining     
|==================================================================================================================================================                                   | 81% ~1 s remaining     
|====================================================================================================================================================                                 | 82% ~1 s remaining     
|========================================================================================================================================================                             | 84% ~1 s remaining     
|==========================================================================================================================================================                           | 85% ~1 s remaining     
|==============================================================================================================================================================                       | 88% ~0 s remaining     
|================================================================================================================================================================                     | 89% ~0 s remaining     
|====================================================================================================================================================================                 | 91% ~0 s remaining     
|======================================================================================================================================================================               | 92% ~0 s remaining     
|==========================================================================================================================================================================           | 94% ~0 s remaining     
|============================================================================================================================================================================         | 95% ~0 s remaining     
|================================================================================================================================================================================     | 98% ~0 s remaining     
|==================================================================================================================================================================================   | 99% ~0 s remaining     

|===========================================================================================================================                                                                    | 65% ~1 s remaining     
|================================================================================================================================                                                               | 67% ~1 s remaining     
|======================================================================================================================================                                                         | 70% ~1 s remaining     
|=============================================================================================================================================                                                  | 74% ~1 s remaining     
|=================================================================================================================================================                                              | 76% ~1 s remaining     
|=====================================================================================================================================================                                          | 78% ~1 s remaining     
|============================================================================================================================================================                                   | 82% ~1 s remaining     
|==================================================================================================================================================================                             | 85% ~0 s remaining     
|=========================================================================================================================================================================                      | 89% ~0 s remaining     
|===============================================================================================================================================================================                | 92% ~0 s remaining     
|====================================================================================================================================================================================           | 94% ~0 s remaining     
|======================================================================================================================================================================================         | 95% ~0 s remaining     
|============================================================================================================================================================================================   | 99% ~0 s remaining     
|=========================================================================================================================                                                                      | 64% ~1 s remaining     
|=============================================================================================================================                                                                  | 66% ~1 s remaining     
|================================================================================================================================                                                               | 67% ~1 s remaining     
|====================================================================================================================================                                                           | 69% ~1 s remaining     
|======================================================================================================================================                                                         | 70% ~1 s remaining     
|=============================================================================================================================================                                                  | 74% ~1 s remaining     
|=================================================================================================================================================                                              | 76% ~1 s remaining     
|=====================================================================================================================================================                                          | 78% ~1 s remaining     
|==========================================================================================================================================================                                     | 81% ~1 s remaining     
|============================================================================================================================================================                                   | 82% ~1 s remaining     
|================================================================================================================================================================                               | 84% ~1 s remaining     
|==================================================================================================================================================================                             | 85% ~1 s remaining     
|=======================================================================================================================================================================                        | 88% ~0 s remaining     
|=========================================================================================================================================================================                      | 89% ~0 s remaining     
|=============================================================================================================================================================================                  | 91% ~0 s remaining     
|===============================================================================================================================================================================                | 92% ~0 s remaining     
|====================================================================================================================================================================================           | 94% ~0 s remaining     
|======================================================================================================================================================================================         | 95% ~0 s remaining     
|==========================================================================================================================================================================================     | 98% ~0 s remaining     
|============================================================================================================================================================================================   | 99% ~0 s remaining     

|==========================================================================================================                                                                                     | 56% ~2 s remaining     
|============================================================================================================                                                                                   | 57% ~2 s remaining     
|===================================================================================================================                                                                            | 60% ~1 s remaining     
|=======================================================================================================================                                                                        | 62% ~1 s remaining     
|=========================================================================================================================                                                                      | 64% ~1 s remaining     
|=============================================================================================================================                                                                  | 66% ~1 s remaining     
|================================================================================================================================                                                               | 67% ~1 s remaining     
|======================================================================================================================================                                                         | 70% ~1 s remaining     
|==========================================================================================================================================                                                     | 73% ~1 s remaining     
|=============================================================================================================================================                                                  | 74% ~1 s remaining     
|=================================================================================================================================================                                              | 76% ~1 s remaining     
|=====================================================================================================================================================                                          | 78% ~1 s remaining     
|============================================================================================================================================================                                   | 82% ~1 s remaining     
|==================================================================================================================================================================                             | 85% ~1 s remaining     
|=========================================================================================================================================================================                      | 89% ~0 s remaining     
|===============================================================================================================================================================================                | 92% ~0 s remaining     
|======================================================================================================================================================================================         | 95% ~0 s remaining     
|============================================================================================================================================================================================   | 99% ~0 s remaining     
|===================================================================================================                                                                                            | 52% ~2 s remaining     
|======================================================================================================                                                                                         | 53% ~2 s remaining     
|==========================================================================================================                                                                                     | 56% ~2 s remaining     
|============================================================================================================                                                                                   | 57% ~2 s remaining     
|===================================================================================================================                                                                            | 60% ~2 s remaining     
|=======================================================================================================================                                                                        | 62% ~1 s remaining     
|=========================================================================================================================                                                                      | 64% ~1 s remaining     
|=============================================================================================================================                                                                  | 66% ~1 s remaining     
|================================================================================================================================                                                               | 67% ~1 s remaining     
|====================================================================================================================================                                                           | 69% ~1 s remaining     
|======================================================================================================================================                                                         | 70% ~1 s remaining     
|==========================================================================================================================================                                                     | 73% ~1 s remaining     
|=============================================================================================================================================                                                  | 74% ~1 s remaining     
|=================================================================================================================================================                                              | 76% ~1 s remaining     
|=====================================================================================================================================================                                          | 78% ~1 s remaining     
|==========================================================================================================================================================                                     | 81% ~1 s remaining     
|============================================================================================================================================================                                   | 82% ~1 s remaining     
|================================================================================================================================================================                               | 84% ~1 s remaining     
|==================================================================================================================================================================                             | 85% ~1 s remaining     
|=======================================================================================================================================================================                        | 88% ~0 s remaining     
|=========================================================================================================================================================================                      | 89% ~0 s remaining     
|=============================================================================================================================================================================                  | 91% ~0 s remaining     
|===============================================================================================================================================================================                | 92% ~0 s remaining     
|====================================================================================================================================================================================           | 94% ~0 s remaining     
|======================================================================================================================================================================================         | 95% ~0 s remaining     
|============================================================================================================================================================================================   | 99% ~0 s remaining     

|============================================================================================================                                                                                   | 57% ~2 s remaining     
|================================================================================================================                                                                               | 59% ~1 s remaining     
|===================================================================================================================                                                                            | 60% ~1 s remaining     
|=========================================================================================================================                                                                      | 64% ~1 s remaining     
|================================================================================================================================                                                               | 67% ~1 s remaining     
|======================================================================================================================================                                                         | 70% ~1 s remaining     
|=============================================================================================================================================                                                  | 74% ~1 s remaining     
|===================================================================================================================================================                                            | 77% ~1 s remaining     
|=====================================================================================================================================================                                          | 78% ~1 s remaining     
|============================================================================================================================================================                                   | 82% ~1 s remaining     
|==================================================================================================================================================================                             | 85% ~0 s remaining     
|=========================================================================================================================================================================                      | 89% ~0 s remaining     
|===============================================================================================================================================================================                | 92% ~0 s remaining     
|====================================================================================================================================================================================           | 94% ~0 s remaining     
|======================================================================================================================================================================================         | 95% ~0 s remaining     
|============================================================================================================================================================================================   | 99% ~0 s remaining     
|=================================================================================================                                                                                              | 51% ~2 s remaining     
|======================================================================================================                                                                                         | 53% ~2 s remaining     
|==========================================================================================================                                                                                     | 56% ~2 s remaining     
|============================================================================================================                                                                                   | 57% ~2 s remaining     
|================================================================================================================                                                                               | 59% ~2 s remaining     
|===================================================================================================================                                                                            | 60% ~2 s remaining     
|=======================================================================================================================                                                                        | 62% ~1 s remaining     
|=========================================================================================================================                                                                      | 64% ~1 s remaining     
|=============================================================================================================================                                                                  | 66% ~1 s remaining     
|================================================================================================================================                                                               | 67% ~1 s remaining     
|====================================================================================================================================                                                           | 69% ~1 s remaining     
|======================================================================================================================================                                                         | 70% ~1 s remaining     
|==========================================================================================================================================                                                     | 73% ~1 s remaining     
|=============================================================================================================================================                                                  | 74% ~1 s remaining     
|=================================================================================================================================================                                              | 76% ~1 s remaining     
|=====================================================================================================================================================                                          | 78% ~1 s remaining     
|==========================================================================================================================================================                                     | 81% ~1 s remaining     
|============================================================================================================================================================                                   | 82% ~1 s remaining     
|================================================================================================================================================================                               | 84% ~1 s remaining     
|==================================================================================================================================================================                             | 85% ~1 s remaining     
|=======================================================================================================================================================================                        | 88% ~0 s remaining     
|=========================================================================================================================================================================                      | 89% ~0 s remaining     
|=============================================================================================================================================================================                  | 91% ~0 s remaining     
|===============================================================================================================================================================================                | 92% ~0 s remaining     
|====================================================================================================================================================================================           | 94% ~0 s remaining     
|======================================================================================================================================================================================         | 95% ~0 s remaining     
|==========================================================================================================================================================================================     | 98% ~0 s remaining     
|============================================================================================================================================================================================   | 99% ~0 s remaining     

|=========================================================================================================================                                                                      | 64% ~1 s remaining     
|=============================================================================================================================                                                                  | 66% ~1 s remaining     
|================================================================================================================================                                                               | 67% ~1 s remaining     
|====================================================================================================================================                                                           | 69% ~1 s remaining     
|======================================================================================================================================                                                         | 70% ~1 s remaining     
|==========================================================================================================================================                                                     | 73% ~1 s remaining     
|=============================================================================================================================================                                                  | 74% ~1 s remaining     
|=================================================================================================================================================                                              | 76% ~1 s remaining     
|=====================================================================================================================================================                                          | 78% ~1 s remaining     
|==========================================================================================================================================================                                     | 81% ~1 s remaining     
|============================================================================================================================================================                                   | 82% ~1 s remaining     
|================================================================================================================================================================                               | 84% ~1 s remaining     
|==================================================================================================================================================================                             | 85% ~0 s remaining     
|=======================================================================================================================================================================                        | 88% ~0 s remaining     
|=========================================================================================================================================================================                      | 89% ~0 s remaining     
|=============================================================================================================================================================================                  | 91% ~0 s remaining     
|===============================================================================================================================================================================                | 92% ~0 s remaining     
|====================================================================================================================================================================================           | 94% ~0 s remaining     
|======================================================================================================================================================================================         | 95% ~0 s remaining     
|========================================================================================================================================================================================       | 97% ~0 s remaining     
|============================================================================================================================================================================================   | 99% ~0 s remaining     
|=========================================================================================================================                                                                      | 64% ~1 s remaining     
|=============================================================================================================================                                                                  | 66% ~1 s remaining     
|================================================================================================================================                                                               | 67% ~1 s remaining     
|====================================================================================================================================                                                           | 69% ~1 s remaining     
|======================================================================================================================================                                                         | 70% ~1 s remaining     
|==========================================================================================================================================                                                     | 73% ~1 s remaining     
|=============================================================================================================================================                                                  | 74% ~1 s remaining     
|=================================================================================================================================================                                              | 76% ~1 s remaining     
|=====================================================================================================================================================                                          | 78% ~1 s remaining     
|==========================================================================================================================================================                                     | 81% ~1 s remaining     
|============================================================================================================================================================                                   | 82% ~1 s remaining     
|================================================================================================================================================================                               | 84% ~1 s remaining     
|==================================================================================================================================================================                             | 85% ~0 s remaining     
|=========================================================================================================================================================================                      | 89% ~0 s remaining     
|=============================================================================================================================================================================                  | 91% ~0 s remaining     
|===============================================================================================================================================================================                | 92% ~0 s remaining     
|====================================================================================================================================================================================           | 94% ~0 s remaining     
|======================================================================================================================================================================================         | 95% ~0 s remaining     
|==========================================================================================================================================================================================     | 98% ~0 s remaining     
|============================================================================================================================================================================================   | 99% ~0 s remaining     

|=======================================================================================================================                                                                        | 62% ~1 s remaining     
|=========================================================================================================================                                                                      | 64% ~1 s remaining     
|================================================================================================================================                                                               | 67% ~1 s remaining     
|======================================================================================================================================                                                         | 70% ~1 s remaining     
|=============================================================================================================================================                                                  | 74% ~1 s remaining     
|===================================================================================================================================================                                            | 77% ~1 s remaining     
|=====================================================================================================================================================                                          | 78% ~1 s remaining     
|============================================================================================================================================================                                   | 82% ~1 s remaining     
|==================================================================================================================================================================                             | 85% ~1 s remaining     
|=========================================================================================================================================================================                      | 89% ~0 s remaining     
|=============================================================================================================================================================================                  | 91% ~0 s remaining     
|===============================================================================================================================================================================                | 92% ~0 s remaining     
|======================================================================================================================================================================================         | 95% ~0 s remaining     
|============================================================================================================================================================================================   | 99% ~0 s remaining     
|=========================================================================================================================                                                                      | 64% ~1 s remaining     
|=============================================================================================================================                                                                  | 66% ~1 s remaining     
|================================================================================================================================                                                               | 67% ~1 s remaining     
|====================================================================================================================================                                                           | 69% ~1 s remaining     
|======================================================================================================================================                                                         | 70% ~1 s remaining     
|==========================================================================================================================================                                                     | 73% ~1 s remaining     
|=============================================================================================================================================                                                  | 74% ~1 s remaining     
|=================================================================================================================================================                                              | 76% ~1 s remaining     
|=====================================================================================================================================================                                          | 78% ~1 s remaining     
|==========================================================================================================================================================                                     | 81% ~1 s remaining     
|============================================================================================================================================================                                   | 82% ~1 s remaining     
|================================================================================================================================================================                               | 84% ~1 s remaining     
|==================================================================================================================================================================                             | 85% ~1 s remaining     
|=======================================================================================================================================================================                        | 88% ~0 s remaining     
|=========================================================================================================================================================================                      | 89% ~0 s remaining     
|=============================================================================================================================================================================                  | 91% ~0 s remaining     
|===============================================================================================================================================================================                | 92% ~0 s remaining     
|====================================================================================================================================================================================           | 94% ~0 s remaining     
|======================================================================================================================================================================================         | 95% ~0 s remaining     
|==========================================================================================================================================================================================     | 98% ~0 s remaining     
|============================================================================================================================================================================================   | 99% ~0 s remaining     

|==============================================================================================================                                                                                 | 58% ~1 s remaining     
|===================================================================================================================                                                                            | 60% ~1 s remaining     
|=========================================================================================================================                                                                      | 64% ~1 s remaining     
|=============================================================================================================================                                                                  | 66% ~1 s remaining     
|================================================================================================================================                                                               | 67% ~1 s remaining     
|======================================================================================================================================                                                         | 70% ~1 s remaining     
|==========================================================================================================================================                                                     | 73% ~1 s remaining     
|=============================================================================================================================================                                                  | 74% ~1 s remaining     
|===================================================================================================================================================                                            | 77% ~1 s remaining     
|=====================================================================================================================================================                                          | 78% ~1 s remaining     
|==========================================================================================================================================================                                     | 81% ~1 s remaining     
|============================================================================================================================================================                                   | 82% ~1 s remaining     
|==================================================================================================================================================================                             | 85% ~1 s remaining     
|=======================================================================================================================================================================                        | 88% ~0 s remaining     
|=========================================================================================================================================================================                      | 89% ~0 s remaining     
|===============================================================================================================================================================================                | 92% ~0 s remaining     
|====================================================================================================================================================================================           | 94% ~0 s remaining     
|======================================================================================================================================================================================         | 95% ~0 s remaining     
|============================================================================================================================================================================================   | 99% ~0 s remaining     
|============================================================================================================                                                                                   | 57% ~2 s remaining     
|================================================================================================================                                                                               | 59% ~1 s remaining     
|===================================================================================================================                                                                            | 60% ~1 s remaining     
|=======================================================================================================================                                                                        | 62% ~1 s remaining     
|=========================================================================================================================                                                                      | 64% ~1 s remaining     
|=============================================================================================================================                                                                  | 66% ~1 s remaining     
|================================================================================================================================                                                               | 67% ~1 s remaining     
|====================================================================================================================================                                                           | 69% ~1 s remaining     
|======================================================================================================================================                                                         | 70% ~1 s remaining     
|==========================================================================================================================================                                                     | 73% ~1 s remaining     
|=============================================================================================================================================                                                  | 74% ~1 s remaining     
|=================================================================================================================================================                                              | 76% ~1 s remaining     
|=====================================================================================================================================================                                          | 78% ~1 s remaining     
|==========================================================================================================================================================                                     | 81% ~1 s remaining     
|============================================================================================================================================================                                   | 82% ~1 s remaining     
|================================================================================================================================================================                               | 84% ~1 s remaining     
|==================================================================================================================================================================                             | 85% ~1 s remaining     
|=======================================================================================================================================================================                        | 88% ~0 s remaining     
|=========================================================================================================================================================================                      | 89% ~0 s remaining     
|=============================================================================================================================================================================                  | 91% ~0 s remaining     
|===============================================================================================================================================================================                | 92% ~0 s remaining     
|====================================================================================================================================================================================           | 94% ~0 s remaining     
|======================================================================================================================================================================================         | 95% ~0 s remaining     
|============================================================================================================================================================================================   | 99% ~0 s remaining     

|==========================================================================================================                                                                                     | 56% ~2 s remaining     
|============================================================================================================                                                                                   | 57% ~2 s remaining     
|===================================================================================================================                                                                            | 60% ~1 s remaining     
|=========================================================================================================================                                                                      | 64% ~1 s remaining     
|================================================================================================================================                                                               | 67% ~1 s remaining     
|======================================================================================================================================                                                         | 70% ~1 s remaining     
|=============================================================================================================================================                                                  | 74% ~1 s remaining     
|===================================================================================================================================================                                            | 77% ~1 s remaining     
|=====================================================================================================================================================                                          | 78% ~1 s remaining     
|============================================================================================================================================================                                   | 82% ~1 s remaining     
|==================================================================================================================================================================                             | 85% ~1 s remaining     
|=========================================================================================================================================================================                      | 89% ~0 s remaining     
|===============================================================================================================================================================================                | 92% ~0 s remaining     
|======================================================================================================================================================================================         | 95% ~0 s remaining     
|============================================================================================================================================================================================   | 99% ~0 s remaining     
|======================================================================================================                                                                                         | 53% ~2 s remaining     
|==========================================================================================================                                                                                     | 56% ~2 s remaining     
|============================================================================================================                                                                                   | 57% ~2 s remaining     
|================================================================================================================                                                                               | 59% ~2 s remaining     
|===================================================================================================================                                                                            | 60% ~1 s remaining     
|=======================================================================================================================                                                                        | 62% ~1 s remaining     
|=========================================================================================================================                                                                      | 64% ~1 s remaining     
|=============================================================================================================================                                                                  | 66% ~1 s remaining     
|================================================================================================================================                                                               | 67% ~1 s remaining     
|====================================================================================================================================                                                           | 69% ~1 s remaining     
|======================================================================================================================================                                                         | 70% ~1 s remaining     
|==========================================================================================================================================                                                     | 73% ~1 s remaining     
|=============================================================================================================================================                                                  | 74% ~1 s remaining     
|=================================================================================================================================================                                              | 76% ~1 s remaining     
|=====================================================================================================================================================                                          | 78% ~1 s remaining     
|==========================================================================================================================================================                                     | 81% ~1 s remaining     
|============================================================================================================================================================                                   | 82% ~1 s remaining     
|================================================================================================================================================================                               | 84% ~1 s remaining     
|==================================================================================================================================================================                             | 85% ~1 s remaining     
|=======================================================================================================================================================================                        | 88% ~0 s remaining     
|=========================================================================================================================================================================                      | 89% ~0 s remaining     
|=============================================================================================================================================================================                  | 91% ~0 s remaining     
|===============================================================================================================================================================================                | 92% ~0 s remaining     
|====================================================================================================================================================================================           | 94% ~0 s remaining     
|======================================================================================================================================================================================         | 95% ~0 s remaining     
|==========================================================================================================================================================================================     | 98% ~0 s remaining     
|============================================================================================================================================================================================   | 99% ~0 s remaining     

|===================================================================================================================                                                                            | 60% ~1 s remaining     
|=========================================================================================================================                                                                      | 64% ~1 s remaining     
|================================================================================================================================                                                               | 67% ~1 s remaining     
|======================================================================================================================================                                                         | 70% ~1 s remaining     
|=============================================================================================================================================                                                  | 74% ~1 s remaining     
|===================================================================================================================================================                                            | 77% ~1 s remaining     
|=====================================================================================================================================================                                          | 78% ~1 s remaining     
|============================================================================================================================================================                                   | 82% ~1 s remaining     
|==================================================================================================================================================================                             | 85% ~0 s remaining     
|=========================================================================================================================================================================                      | 89% ~0 s remaining     
|===============================================================================================================================================================================                | 92% ~0 s remaining     
|======================================================================================================================================================================================         | 95% ~0 s remaining     
|============================================================================================================================================================================================   | 99% ~0 s remaining     
|=========================================================================================================================                                                                      | 64% ~1 s remaining     
|=============================================================================================================================                                                                  | 66% ~1 s remaining     
|================================================================================================================================                                                               | 67% ~1 s remaining     
|====================================================================================================================================                                                           | 69% ~1 s remaining     
|======================================================================================================================================                                                         | 70% ~1 s remaining     
|==========================================================================================================================================                                                     | 73% ~1 s remaining     
|=============================================================================================================================================                                                  | 74% ~1 s remaining     
|=================================================================================================================================================                                              | 76% ~1 s remaining     
|=====================================================================================================================================================                                          | 78% ~1 s remaining     
|==========================================================================================================================================================                                     | 81% ~1 s remaining     
|============================================================================================================================================================                                   | 82% ~1 s remaining     
|================================================================================================================================================================                               | 84% ~1 s remaining     
|==================================================================================================================================================================                             | 85% ~0 s remaining     
|=======================================================================================================================================================================                        | 88% ~0 s remaining     
|=========================================================================================================================================================================                      | 89% ~0 s remaining     
|=============================================================================================================================================================================                  | 91% ~0 s remaining     
|===============================================================================================================================================================================                | 92% ~0 s remaining     
|====================================================================================================================================================================================           | 94% ~0 s remaining     
|======================================================================================================================================================================================         | 95% ~0 s remaining     
|==========================================================================================================================================================================================     | 98% ~0 s remaining     
|============================================================================================================================================================================================   | 99% ~0 s remaining     

|=============================================================================================================================                                                                  | 66% ~1 s remaining     
|================================================================================================================================                                                               | 67% ~1 s remaining     
|==================================================================================================================================                                                             | 68% ~1 s remaining     
|======================================================================================================================================                                                         | 70% ~1 s remaining     
|=============================================================================================================================================                                                  | 74% ~1 s remaining     
|===================================================================================================================================================                                            | 77% ~1 s remaining     
|=====================================================================================================================================================                                          | 78% ~1 s remaining     
|============================================================================================================================================================                                   | 82% ~1 s remaining     
|==================================================================================================================================================================                             | 85% ~1 s remaining     
|=========================================================================================================================================================================                      | 89% ~0 s remaining     
|===============================================================================================================================================================================                | 92% ~0 s remaining     
|======================================================================================================================================================================================         | 95% ~0 s remaining     
|============================================================================================================================================================================================   | 99% ~0 s remaining     
|=========================================================================================================================                                                                      | 64% ~1 s remaining     
|=============================================================================================================================                                                                  | 66% ~1 s remaining     
|================================================================================================================================                                                               | 67% ~1 s remaining     
|====================================================================================================================================                                                           | 69% ~1 s remaining     
|======================================================================================================================================                                                         | 70% ~1 s remaining     
|==========================================================================================================================================                                                     | 73% ~1 s remaining     
|=============================================================================================================================================                                                  | 74% ~1 s remaining     
|=================================================================================================================================================                                              | 76% ~1 s remaining     
|=====================================================================================================================================================                                          | 78% ~1 s remaining     
|==========================================================================================================================================================                                     | 81% ~1 s remaining     
|============================================================================================================================================================                                   | 82% ~1 s remaining     
|================================================================================================================================================================                               | 84% ~1 s remaining     
|==================================================================================================================================================================                             | 85% ~0 s remaining     
|=======================================================================================================================================================================                        | 88% ~0 s remaining     
|=========================================================================================================================================================================                      | 89% ~0 s remaining     
|=============================================================================================================================================================================                  | 91% ~0 s remaining     
|===============================================================================================================================================================================                | 92% ~0 s remaining     
|====================================================================================================================================================================================           | 94% ~0 s remaining     
|======================================================================================================================================================================================         | 95% ~0 s remaining     
|==========================================================================================================================================================================================     | 98% ~0 s remaining     
|============================================================================================================================================================================================   | 99% ~0 s remaining     
#write.csv(full_residuals, "../data/analysis_checkpoints/full_residuals.csv")
#write.csv(baseline_residuals, "../data/analysis_checkpoints/baseline_residuals.csv")
model_deltas = log_lik_deltas %>%
  group_by(model, training, seed, corpus) %>% 
  summarise(mean_delta_log_lik = mean(delta_log_lik),
            sem_delta_log_lik = sd(delta_log_lik) / sqrt(length(delta_log_lik)))
write.csv(full_model_results, "../data/analysis_checkpoints/full_model_result.csv")
write.csv(baseline_results, "../data/analysis_checkpoints/baseline_results.csv")
#full_model_results = read.csv("../data/analysis_checkpoints/ffull_model_results.csv")
#baseline_results = read.csv("../data/analysis_checkpoints/fbaseline_resultsb.csv")
metric <- "ΔLogLik"
#metric <- "-ΔMSE"

# # Select the relevant metric.
model_deltas = model_deltas %>%
    # Retrieve the current test metric
    mutate(delta_test_mean = mean_delta_log_lik,
           delta_test_sem = sem_delta_log_lik) %>%
    # mutate(delta_test_mean = mean_delta_mse,
    #        delta_test_sem = sem_delta_mse)
    
    # Remove the raw metrics.
    select(-mean_delta_log_lik, -sem_delta_log_lik,
           #-mean_delta_mse, -sem_delta_mse
           )
model_deltas
# Sanity check: training on train+test data should yield improved performance over training on just training data. (When evaluating on test data.)
# full_baselines = all_data %>%
#   group_by(model, training, seed, corpus) %>%
#   summarise(baseline_train_all_test_lik = logLik_test(lm(psychometric ~ len + freq + sent_pos, data=.), semi_join(test_data, ., by=c("training", "model", "seed", "corpus")), semi_join(test_data, ., by=c("training", "model", "seed", "corpus"))$psychometric)) %>%
#   ungroup()
# full_baselines
# 
# full_baselines %>%
#   right_join(baselines, by=c("seed", "training", "model", "corpus")) %>%
#   mutate(delta=baseline_train_all_test_lik-baseline_test_lik) %>%
#   select(-baseline_lik) # %>%
#   #select(-baseline_test_lik, -baseline_train_all_test_lik, -baseline_lik, -baseline_test_mse)

Load language model data (SyntaxGym, PPL)

language_model_data = read.csv("../data/model_metadata.csv") %>%
  mutate(model = as.character(model),
         model = if_else(model == "gpt-2", "gpt2", model),
         model = as.factor(model)) %>%
  mutate(train_size = case_when(str_starts(training, "bllip-lg") ~ 42,
                                str_starts(training, "bllip-md") ~ 15,
                                str_starts(training, "bllip-sm") ~ 5,
                                str_starts(training, "bllip-xs") ~ 1),
         
         # Training vocabulary usually covaries with the training corpus.
         # But BPE models share a vocabulary across training corpora.
         training_vocab=as.factor(ifelse(str_detect(training, "gptbpe"), "gptbpe", as.character(training))),
         training_source=as.factor(str_replace(as.character(training), "-gptbpe", ""))
         ) %>%
  mutate(seed = as.factor(seed)) %>%
  select(-pid, -test_loss) %>%
  distinct(model, training, seed, .keep_all = TRUE)
table(language_model_data$seed)

         0        111        120        922       1111       3602       4301       7245       7877      28066      28068      44862      51272      64924 1581807512 1581807578 1581861474 1581955288 
         4          7          6          5          4          1          1          1          1          1          1          1          1          1          1          1          1          1 
1582126320 1586986276 1587139950 
         1          1          1 
table(model_deltas$seed)

       111        120        607        922       1111       3602       4301       7245       7877      28066      28068      44862      51272      64924 1581807512 1581807578 1581861474 1581955288 
         9          9          1          9         12          3          3          3          3          3          3          3          3          3          3          3          3          3 
1582126320 1586986276 1587139950 
         3          3          3 

First join delta-metric data with model auxiliary data.

model_deltas = model_deltas %>%
  merge(language_model_data, by = c("seed", "training", "model"), all=T) %>%
  drop_na()

model_deltas

Also join on the original linear model data, rather than collapsing to delta-metrics. This will support regressions later on that don’t collapse across folds.

Final data preprocessing

# Exclude ordered-neurons from all analyses.
model_deltas <- model_deltas %>%
  filter(model != "ordered-neurons")

Visualizations

The basics

all_data %>% ggplot(aes(x=corpus)) + geom_bar()

print(all_data %>% group_by(corpus) %>% summarise(n=n()))
all_data %>% 
  ggplot(aes(x=freq, color=corpus)) + geom_density()

all_data %>% 
  ggplot(aes(x=len, color=corpus)) + geom_density()

all_data %>% 
  ggplot(aes(x=surprisal, color=corpus)) + geom_density()

Predictive power and SG

model_deltas %>%
  ggplot(aes(x=sg_score, y=delta_test_mean)) +
    geom_errorbar(aes(ymin=delta_test_mean-delta_test_sem, ymax=delta_test_mean+delta_test_sem)) +
    geom_smooth(method="lm", se=T) +
    geom_point(stat="identity", position="dodge", alpha=1, size=3, aes(color=training_vocab, shape=model)) +
    ylab(metric) +
    xlab("Syntax Generalization Score") +
    ggtitle("Syntactic Generalization vs. Predictive Power") +
    scale_color_manual(values = c("bllip-lg"="#440154FF",
                              "bllip-md"="#39568CFF",
                              "bllip-sm"="#1F968BFF",
                              "bllip-xs"="#73D055FF",
                              "gptbpe"="#888888")) +
    facet_grid(~corpus, scales="free") +
    theme(axis.text=element_text(size=14),
          strip.text.x = element_text(size=14),
          legend.text=element_text(size=14),
          axis.title=element_text(size=18),
          legend.position = "bottom")

#ggsave("./cogsci_images/sg_loglik.png",height=5,width=6)

Regression analyses

We control for effects of perplexity by relating the residuals of a performance ~ PPL regression to SG score.

# Prepare a residualized regression for x1 onto y, controlling for the effects of x2.
d_resid = model_deltas %>%
  drop_na() %>%
  
  group_by(corpus) %>%
    # Residualize delta metric w.r.t PPL for each model--training--seed within
    # training vocabulary
    mutate(resid.delta = resid(lm(delta_test_mean ~ training_vocab:test_ppl))) %>%
    # Residualize SG score w.r.t. PPL for each model--training--seed
    # within training vocabulary
    mutate(resid.sg = resid(lm(sg_score ~ training_vocab:test_ppl))) %>%
  ungroup()
  
  # # Compute summary statistics across model--training--seed--corpus.
  # group_by(model, training_vocab, corpus, seed) %>%
  #   summarise(resid.delta.mean = mean(resid.delta),
  #             resid.delta.sem = sd(resid.delta) / sqrt(length(resid.delta)),
  #             resid.sg.mean = mean(resid.sg),
  #             resid.sg.sem = sd(resid.sg) / sqrt(length(resid.sg))) %>%
  # ungroup()

# Now plot residual vs SG
d_resid %>%
  #filter(corpus != "bnc-brown") %>%
  ggplot(aes(x=resid.sg, y=resid.delta)) +
    theme_bw() +
    scale_shape_manual(values = c(16, 17, 15, 18)) +
    geom_smooth(method="lm", se=T, alpha=0.3) +
    geom_point(stat="identity", position="dodge", alpha=1, size=4, aes(shape=model, color=training_vocab)) +
    ylab(paste("Residual", metric)) +
    xlab("Residual Syntax Generalization Score") +
    ggtitle("Syntactic Generalization vs. Predictive Power") +
    scale_color_manual(values = c("bllip-lg"="#440154FF",
                                  "bllip-md"="#39568CFF",
                                  "bllip-sm"="#1F968BFF",
                                  "bllip-xs"="#73D055FF",
                                  "gptbpe"="#f0941f")) +
    facet_grid(.~corpus, scales="free") +
    theme(axis.text=element_text(size=14),
          strip.text.x = element_text(size=14),
          legend.text=element_text(size=14),
          axis.title=element_text(size=18),
          legend.position = "right")
ggsave("../images/cuny2020/dll_sg.png",height=4.5,width=9)

do_stepwise_regression = function(cur_corpus) {
  regression_data = model_deltas %>%
    filter(corpus == cur_corpus)
  
  print("----------------------")
  print(cur_corpus)
  
  lm1 = lm(delta_test_mean ~ training_vocab:test_ppl, data = regression_data)
  lm2 = lm(delta_test_mean ~ training_vocab:test_ppl + sg_score, data = regression_data)
  print(anova(lm1, lm2))
  summary(lm2)
}
do_stepwise_regression("bnc-brown")
[1] "----------------------"
[1] "bnc-brown"
Analysis of Variance Table

Model 1: delta_test_mean ~ training_vocab:test_ppl
Model 2: delta_test_mean ~ training_vocab:test_ppl + sg_score
  Res.Df        RSS Df Sum of Sq     F Pr(>F)
1     23 0.00024405                          
2     22 0.00023954  1 4.508e-06 0.414 0.5266

Call:
lm(formula = delta_test_mean ~ training_vocab:test_ppl + sg_score, 
    data = regression_data)

Residuals:
       Min         1Q     Median         3Q        Max 
-0.0049611 -0.0018306 -0.0005214  0.0006757  0.0064101 

Coefficients:
                                  Estimate Std. Error t value Pr(>|t|)   
(Intercept)                      1.316e-02  3.624e-03   3.632  0.00147 **
sg_score                         3.321e-03  5.161e-03   0.643  0.52658   
training_vocabbllip-lg:test_ppl  1.925e-05  3.358e-05   0.573  0.57227   
training_vocabbllip-md:test_ppl -4.247e-05  2.799e-05  -1.518  0.14336   
training_vocabbllip-sm:test_ppl -5.112e-05  2.492e-05  -2.051  0.05233 . 
training_vocabbllip-xs:test_ppl -6.235e-05  1.663e-05  -3.750  0.00111 **
training_vocabgptbpe:test_ppl   -1.119e-05  8.169e-06  -1.370  0.18450   
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 0.0033 on 22 degrees of freedom
Multiple R-squared:  0.642, Adjusted R-squared:  0.5444 
F-statistic: 6.576 on 6 and 22 DF,  p-value: 0.0004362
do_stepwise_regression("dundee")
[1] "----------------------"
[1] "dundee"
Analysis of Variance Table

Model 1: delta_test_mean ~ training_vocab:test_ppl
Model 2: delta_test_mean ~ training_vocab:test_ppl + sg_score
  Res.Df        RSS Df  Sum of Sq     F Pr(>F)
1     23 9.6740e-05                           
2     22 9.6289e-05  1 4.5086e-07 0.103 0.7513

Call:
lm(formula = delta_test_mean ~ training_vocab:test_ppl + sg_score, 
    data = regression_data)

Residuals:
       Min         1Q     Median         3Q        Max 
-0.0025554 -0.0011305 -0.0007635  0.0011585  0.0038662 

Coefficients:
                                  Estimate Std. Error t value Pr(>|t|)   
(Intercept)                      7.120e-03  2.298e-03   3.098  0.00525 **
sg_score                         1.050e-03  3.272e-03   0.321  0.75127   
training_vocabbllip-lg:test_ppl -1.188e-05  2.129e-05  -0.558  0.58250   
training_vocabbllip-md:test_ppl -1.813e-05  1.774e-05  -1.022  0.31796   
training_vocabbllip-sm:test_ppl -2.308e-05  1.580e-05  -1.461  0.15826   
training_vocabbllip-xs:test_ppl -2.271e-05  1.054e-05  -2.154  0.04243 * 
training_vocabgptbpe:test_ppl   -1.894e-06  5.179e-06  -0.366  0.71816   
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 0.002092 on 22 degrees of freedom
Multiple R-squared:  0.3203,    Adjusted R-squared:  0.135 
F-statistic: 1.728 on 6 and 22 DF,  p-value: 0.1615
do_stepwise_regression("natural-stories")
[1] "----------------------"
[1] "natural-stories"
Analysis of Variance Table

Model 1: delta_test_mean ~ training_vocab:test_ppl
Model 2: delta_test_mean ~ training_vocab:test_ppl + sg_score
  Res.Df        RSS Df  Sum of Sq      F Pr(>F)
1     23 3.9684e-05                            
2     22 3.7375e-05  1 2.3089e-06 1.3591 0.2562

Call:
lm(formula = delta_test_mean ~ training_vocab:test_ppl + sg_score, 
    data = regression_data)

Residuals:
       Min         1Q     Median         3Q        Max 
-0.0022223 -0.0008183  0.0002092  0.0009282  0.0023418 

Coefficients:
                                  Estimate Std. Error t value Pr(>|t|)    
(Intercept)                      1.076e-02  1.432e-03   7.512 1.65e-07 ***
sg_score                        -2.377e-03  2.039e-03  -1.166   0.2562    
training_vocabbllip-lg:test_ppl -6.333e-05  1.326e-05  -4.774 9.11e-05 ***
training_vocabbllip-md:test_ppl -6.220e-05  1.105e-05  -5.627 1.17e-05 ***
training_vocabbllip-sm:test_ppl -6.636e-05  9.844e-06  -6.742 8.93e-07 ***
training_vocabbllip-xs:test_ppl -3.823e-05  6.568e-06  -5.821 7.41e-06 ***
training_vocabgptbpe:test_ppl   -8.201e-06  3.227e-06  -2.541   0.0186 *  
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 0.001303 on 22 degrees of freedom
Multiple R-squared:  0.7632,    Adjusted R-squared:  0.6986 
F-statistic: 11.82 on 6 and 22 DF,  p-value: 6.283e-06
# The residualized analysis and the stepwise regression analysis
# should yield the same coefficients for the SG score variable.
#
# Below, we compute the slope coefficient for the SG term in the
# residualized analyses.
#
# These coefficients should match those found in the stepwise
# regression for `sg_score` above.
d_resid %>% group_by(corpus) %>%
  group_modify(~tidy(lm(resid.delta ~ training_vocab:test_ppl + resid.sg, data=.))
                 %>% filter(term == "resid.sg")) %>% 
  select(corpus, estimate)

Predictive power and perplexity

model_deltas %>%
  mutate(test_ppl = if_else(test_ppl > 500, 329.9, test_ppl)) %>%
  ggplot(aes(x=test_ppl, y=delta_test_mean, color=training_vocab, fill = training_vocab, ymin=0)) +
    theme_bw() +
    geom_text(aes(x=275, y=0, label = c("//"))) +
    geom_errorbar(aes(ymin=delta_test_mean-delta_test_sem, ymax=delta_test_mean+delta_test_sem), alpha=0.4) +
    #geom_smooth(method="lm", se=F) +
    geom_point(stat="identity", position="dodge", alpha=1, size=4, aes(shape=model, color = training_vocab)) +
    ylab(metric) +
    xlab("Test Perplexity") +
    #coord_cartesian(ylim = c(1, 16)) +
    ggtitle("Test Perplexity vs. Predictive Power") +
    scale_color_manual(values = c("bllip-lg"="#440154FF",
                                  "bllip-md"="#39568CFF",
                                  "bllip-sm"="#1F968BFF",
                                  "bllip-xs"="#73D055FF",
                                  "gptbpe"="#f0941f")) +
    scale_shape_manual(values = c(16, 17, 15, 18)) +
    scale_x_continuous(labels=c(0, 50, 100, 150, 200, 250, 500 ,550), breaks=c(0, 50, 100, 150, 200, 250, 300, 350), minor_breaks = NULL) +
    scale_y_continuous(limits = c(0, NA), expand = c(0,0)) +
    facet_wrap(~corpus, scales="free") +
    coord_cartesian(clip="off") +
    theme(axis.text=element_text(size=12),
          strip.text.x = element_text(size=12),
          legend.text=element_text(size=12),
          axis.title=element_text(size=12),
          legend.position = "right")
ggsave("../images/cuny2020/ppl_loglik.png",height=4.2,width=12)


dll_cor_test = function(df){
  df %>%
    summarise(
      cor = cor.test(df$delta_test_mean, df$test_ppl)$estimate,
      p = cor.test(df$delta_test_mean, df$test_ppl)$p.value
    )
}

model_deltas %>%
  filter(model != "5gram") %>%
  group_by(training, corpus) %>%
    mutate(n = n()) %>%
  ungroup() %>%
  filter(n > 2) %>%
  group_by(training, corpus) %>%
    do({ dll_cor_test(.) }) %>%
  ungroup() %>%
  arrange(corpus)
NA

Effect of training data size

model_deltas %>%
  mutate(train_size = log(train_size)) %>%
  mutate(bpe = if_else(training_vocab == "gptbpe", "yes", "no"),
         bpe = as.factor(bpe)) %>%
  ggplot(aes(x=train_size, y=delta_test_mean, color=model)) +
    theme_bw() +
    geom_errorbar(aes(ymin=delta_test_mean-delta_test_sem, ymax=delta_test_mean+delta_test_sem), width = 0.1) +
    geom_smooth(method="lm", se=T, alpha=0.2) +
    geom_point(stat="identity", position="dodge", alpha=1, size=3, aes(shape=bpe)) +
    ylab(metric) +
    xlab("Log Million Training Tokens") +
    ggtitle("Training Size vs. Predictive Power") +
    facet_grid(.~corpus, scales="free") +
    #scale_color_manual(values = c("#A42EF1", "#3894C8")) +
    theme(axis.text=element_text(size=12),
          strip.text.x = element_text(size=12),
          legend.text=element_text(size=8),
          legend.title=element_text(size=8),
          axis.title=element_text(size=14),
          legend.position = "bottom",
          legend.direction = "horizontal",
          legend.key.width = unit(0.3,"cm"),
          legend.spacing.x = unit(0.1, 'cm'))

#ggsave("../images/cuny2020/training_loglik.png",height=5,width=5)

model_cor_test = function(df){
  df %>%
    summarise(cor = cor.test(df$train_size, df$delta_test_mean)$estimate,
              p = cor.test(df$train_size, df$delta_test_mean)$p.value)
}

model_deltas %>%
  group_by(model, corpus) %>%
    do({model_cor_test(.)}) %>%
  ungroup() %>%
  arrange()
NA
NA
model_deltas %>%
  mutate(test_ppl = if_else(test_ppl > 500, 329.9, test_ppl)) %>%
  mutate(train_size = log(train_size)) %>%
  mutate(bpe = if_else(training_vocab == "gptbpe", "yes", "no"),
         bpe = as.factor(bpe)) %>%
  ggplot(aes(x=test_ppl, y=sg_score, color=training_vocab)) +
    theme_bw() +
    #geom_smooth(method="lm", se=T, alpha=0.2) +
    geom_point(stat="identity", position="dodge", alpha=0.6, size=5, aes(shape=model)) +
    geom_text(aes(x=275, y=0, label = c("//"))) +
    ylab("SG SCore") +
    xlab("Test Perplexity") +
    ggtitle("Test PPL vs. SG Score") +
    scale_color_manual(values = c("bllip-lg"="#440154FF",
                                  "bllip-md"="#39568CFF",
                                  "bllip-sm"="#1F968BFF",
                                  "bllip-xs"="#73D055FF",
                                  "gptbpe"="#f0941f")) +
    scale_shape_manual(values = c(16, 17, 15, 18)) +
    scale_x_continuous(labels=c(0, 50, 100, 150, 200, 250, 500 ,550), breaks=c(0, 50, 100, 150, 200, 250, 300, 350), minor_breaks = NULL) +
    scale_y_continuous(limits = c(0, 1), expand = c(0,0)) +
    theme(axis.text=element_text(size=12),
          strip.text.x = element_text(size=12),
          legend.text=element_text(size=8),
          legend.title=element_text(size=8),
          axis.title=element_text(size=14),
          legend.position = "none",
          legend.direction = "horizontal",
          legend.key.width = unit(0.3,"cm"),
          legend.spacing.x = unit(0.1, 'cm'))
ggsave("../images/cuny2020/ppl_sg.png",height=4.5,width=3)

model_deltas %>%
  mutate(train_size = log(train_size)) %>%
  mutate(bpe = if_else(training_vocab == "gptbpe", "yes", "no"),
         bpe = as.factor(bpe)) %>%
  ggplot(aes(x=train_size, y=sg_score, color=model)) +
    theme_bw() +
    geom_smooth(method="lm", se=T, alpha=0.2) +
    geom_point(stat="identity", position="dodge", alpha=1, size=3, aes(shape=bpe)) +
    ylab("SG SCore") +
    xlab("Log Million Training Tokens") +
    ggtitle("Training Size vs. SG Score") +
    #scale_color_manual(values = c("#A42EF1", "#3894C8")) +
    #facet_grid(~model, scales="free") +
    theme(axis.text=element_text(size=12),
          strip.text.x = element_text(size=12),
          legend.text=element_text(size=8),
          legend.title=element_text(size=8),
          axis.title=element_text(size=14),
          legend.position = "bottom",
          legend.direction = "horizontal",
          legend.key.width = unit(0.3,"cm"),
          legend.spacing.x = unit(0.1, 'cm'))

#ggsave("../images/cuny2020/training_sg.png",height=5,width=4)

model_cor_test = function(df){
  df %>%
    summarise(cor = cor.test(df$train_size, df$sg_score)$estimate,
              p = cor.test(df$train_size, df$sg_score)$p.value)
}

model_deltas %>%
  group_by(model) %>%
    do({model_cor_test(.)}) %>%
  ungroup()
NA
NA
NA

Smith & Levy reproduction

This redone so that it’s unique for each model

all_data %>%
  ggplot(aes(x=surprisal, color=model)) +
  theme_bw() +
  geom_density() +
  facet_grid(~corpus) +
  coord_cartesian(xlim = c(0, 21)) +
  theme(panel.spacing = unit(2.5, "cm"))
ggsave("../images/cuny2020/surp_corr_marginals.png",height=1.5,width=11)


smooths %>%
  #mutate(training_model = paste(training, "_", model, sep="")) %>%
  #filter(training_vocab == "bllip-xs" | training_vocab == "gptbpe" | training_vocab == "bllip-lg") %>%
  #filter(training != "bllip-md-gptbpe" & training != "bllip-sm-gptbpe") %>%
  # mutate(training = as.character(training),
  #        training = if_else(training == "bllip-lg-gptbpe", "bpe \n bllip-lg", training),
  #        training = if_else(training == "bllip-md-gptbpe", "bpe \n bllip-md", training),
  #        training = if_else(training == "bllip-sm-gptbpe", "bpe \n bllip-sm", training),
  #        training = if_else(training == "bllip-xs-gptbpe", "bpe \n bllip-xs", training)) %>%
  ggplot(aes(x=surprisal, y=y, fill=training_vocab, linetype=model)) +
      theme_bw() +
      geom_line(size=0.5, aes(color=training_vocab)) +
      #geom_line(aes(y=y_lower), linetype="dashed") +
      geom_ribbon(aes(ymin=y_lower,ymax=y_upper), alpha=0.3) +
      #geom_line(aes(y=y_upper), linetype="dashed") +
      facet_grid(corpus ~ training_vocab + model, scales="free") +
      scale_color_manual(values = c("bllip-lg"="#440154FF",
                                  "bllip-md"="#39568CFF",
                                  "bllip-sm"="#1F968BFF",
                                  "bllip-xs"="#73D055FF",
                                  "gptbpe"="#f0941f")) +
      scale_fill_manual(values = c("bllip-lg"="#440154FF",
                                  "bllip-md"="#39568CFF",
                                  "bllip-sm"="#1F968BFF",
                                  "bllip-xs"="#73D055FF",
                                  "gptbpe"="#f0941f")) +
      scale_x_continuous(labels=c(0, 10, 20), breaks=c(0, 10, 20), minor_breaks = NULL) +
      ylab("Reading Time") +
      theme(legend.position = "bottom")
ggsave("../images/cuny2020/gam_surp_corr.png", height=5,width=12)

NA
NA
NA

Old Plotting

all_data %>%
  filter(model == "gpt2", corpus == "dundee") %>%
  filter(surprisal<21) %>%
  mutate(bpe=str_detect(training, "bpe"),
         training_source=str_replace(training, "-gptbpe", "")) %>% 
  ggplot(aes(x=surprisal, y=psychometric, color=training_source, linetype=bpe)) +
    theme_bw() +
    #stat_smooth(se=T, alpha=0.5) +
    geom_smooth(method = "gam", formula = psychometric ~ s(surprisal, bs = 'cr', k = 20) + s(prev_surp, bs = 'cr', k = 20) + te(freq, len, bs = 'cr') + te(prev_freq, prev_len, bs = 'cr', se = F)) +
    #geom_errorbar(color="black", width=.2, position=position_dodge(width=.9), alpha=0.3) +
    #geom_point(stat="identity", position="dodge", alpha=1, size=3) +
    ylab("Processing Time (ms)") +
    xlab("Surprisal (bits)") +
    ggtitle("Surprisal vs. Reading Time / Gaze Duration") +
    facet_wrap(model ~ corpus, scales="free", ncol=3, strip.position = c("right")) +
    scale_color_manual(values = c("bllip-lg"="#440154FF",
                               "bllip-md"="#39568CFF",
                               "bllip-sm"="#1F968BFF",
                               "bllip-xs"="#73D055FF",
                               "bllip-lg-gptbpe"="#888888",
                               "bllip-md-gptbpe"="#888888",
                               "bllip-sm-gptbpe"="#888888",
                               "bllip-xs-gptbpe"="#888888")) +
    coord_cartesian(xlim = c(0, 21)) +
    theme(axis.text=element_text(size=10),
          axis.text.y = element_text(size = 10),
          strip.text.x = element_text(size=10),
          legend.text=element_text(size=10),
          axis.title=element_text(size=12),
          legend.position = "right")

#ggsave("../images/cuny2020/surp_corr.png",height=6,width=12)

corr_test = function(df){
  df %>%
    summarise(
      cor = cor.test(df$surprisal, df$psychometric)$estimate
    )
}

all_data %>%
  group_by(model, training, corpus, seed) %>%
    do({ cor = corr_test(.)}) %>%
  ungroup()
NA

Investigate vanilla

all_data %>%
  #filter(surprisal < 15, surprisal > 0) %>%
  filter(model == "vanilla") %>% 
  ggplot(aes(x=surprisal, y=psychometric)) +
    #stat_smooth(se=T, alpha=0.5) +
    #geom_errorbar(color="black", width=.2, position=position_dodge(width=.9), alpha=0.3) +
    geom_point(alpha=0.1) + #stat="identity", position="dodge", alpha=1, size=3) +
    ylab("Processing Time (ms)") +
    xlab("Surprisal (bits)") +
    ggtitle("Surprisal vs. Reading Time / Gaze Duration: Vanilla") +
    facet_grid(corpus~training, scales = "free")

    # scale_color_manual(values = c("bllip-lg"="#440154FF",
    #                           "bllip-md"="#39568CFF",
    #                           "bllip-sm"="#1F968BFF",
    #                           "bllip-xs"="#73D055FF",
    #                           "bllip-lg-gptbpe"="#888888",
    #                           "bllip-md-gptbpe"="#888888",
    #                           "bllip-sm-gptbpe"="#888888",
    #                           "bllip-xs-gptbpe"="#888888"))
all_data %>% 
  filter(corpus == "dundee", model == "vanilla", training == "bllip-lg", surprisal > 20, psychometric < 300)
print(full_residuals %>% filter(corpus == "dundee", model == "vanilla", training == "bllip-lg") %>% arrange(desc(resid)))
full_residuals %>% filter(corpus == "dundee", model == "vanilla", training == "bllip-lg") %>% arrange(desc(resid)) %>% filter(resid > 150) %>% 
  ggplot(aes(x=surprisal)) + geom_density()

Investigate RNNG

all_data %>%
  #filter(surprisal < 15, surprisal > 0) %>%
  filter(model == "rnng") %>% 
  ggplot(aes(x=surprisal, y=psychometric)) +
    #stat_smooth(se=T, alpha=0.5) +
    #geom_errorbar(color="black", width=.2, position=position_dodge(width=.9), alpha=0.3) +
    geom_point(alpha=0.1) + #stat="identity", position="dodge", alpha=1, size=3) +
    ylab("Processing Time (ms)") +
    xlab("Surprisal (bits)") +
    ggtitle("Surprisal vs. Reading Time / Gaze Duration: RNNG") +
    facet_grid(corpus~training, scales = "free")

all_data %>% 
  filter(corpus == "dundee", model == "rnng", training == "bllip-lg", surprisal > 20, psychometric < 300)
print(full_residuals %>% filter(corpus == "dundee", model == "rnng", training == "bllip-lg") %>% arrange(desc(resid)))
full_residuals %>% filter(corpus == "dundee", model == "rnng", training == "bllip-lg") %>% arrange(desc(resid)) %>% filter(resid > 150) %>% 
  ggplot(aes(x=surprisal)) + geom_density()

Investigate ngram vs vanilla

ngram_resids = full_residuals %>% filter(model == "5gram", training == "bllip-sm") %>% group_by(corpus, code) %>% summarise(freq=mean(freq), psychometric=mean(psychometric), surprisal=mean(surprisal), resid=mean(resid))
vanilla_resids = full_residuals %>% filter(model == "vanilla", training == "bllip-sm") %>% group_by(corpus, code) %>% summarise(freq=mean(freq), psychometric=mean(psychometric), surprisal=mean(surprisal),  resid=mean(resid))
resids_joined = ngram_resids %>% left_join(vanilla_resids, by=c("corpus", "code"), suffix=c(".ngram", ".vanilla"))

resids_joined %>% 
  ggplot(aes(x=resid.ngram, y=resid.vanilla)) + geom_point() + geom_abline(slope=1, color="red") +
  facet_grid(~corpus)


resids_joined %>% 
  mutate(resid_diff=resid.ngram - resid.vanilla) %>% 
  ggplot(aes(x=resid_diff)) + geom_density() +
  facet_grid(~corpus)


resids_joined %>% 
  mutate(resid_diff=abs(resid.ngram) - abs(resid.vanilla),
         big=resid_diff < -10) %>% 
  ggplot(aes(x=surprisal.ngram, color=big)) + geom_density() + facet_grid(~corpus) +
  ggtitle("ngram surprisal of high-improvement tokens (relative to vanilla)")


resids_joined %>% 
  mutate(resid_abs_diff=abs(resid.ngram - resid.vanilla)) %>% 
  ggplot(aes(x=freq.ngram, y=resid_abs_diff)) + geom_point(alpha=0.1) + geom_smooth()

Investigate gptbpe vs vanilla

gpt_resids = full_residuals %>% filter(model == "gpt2", training == "bllip-sm-gptbpe") %>% group_by(corpus, code) %>% summarise(freq=mean(freq), psychometric=mean(psychometric), surprisal=mean(surprisal),  resid=mean(resid))
vanilla_resids = full_residuals %>% filter(model == "vanilla", training == "bllip-sm") %>% group_by(corpus, code) %>% summarise(freq=mean(freq), psychometric=mean(psychometric), surprisal=mean(surprisal),  resid=mean(resid))
resids_joined = gpt_resids %>% left_join(vanilla_resids, by=c("corpus", "code"), suffix=c(".gpt", ".vanilla"))

resids_joined %>% 
  ggplot(aes(x=resid.gpt, y=resid.vanilla)) + geom_point() + geom_abline(slope=1, color="red") +
  facet_grid(~corpus)


resids_joined %>% 
  mutate(resid_diff=resid.gpt - resid.vanilla) %>% 
  ggplot(aes(x=resid_diff)) + geom_density() +
  facet_grid(~corpus)


resids_joined %>% 
  mutate(resid_diff=abs(resid.gpt) - abs(resid.vanilla),
         big=resid_diff < -10) %>% 
  ggplot(aes(x=surprisal.gpt, color=big)) + geom_density() + facet_grid(~corpus) +
  ggtitle("gpt surprisal of high-improvement tokens (relative to vanilla)")


resids_joined %>% 
  mutate(resid_abs_diff=abs(resid.gpt - resid.vanilla)) %>% 
  ggplot(aes(x=freq.gpt, y=resid_abs_diff)) + geom_point(alpha=0.1) + geom_smooth()

Investigate residuals overall

resid_deltas = full_residuals %>% right_join(baseline_residuals, by=c("corpus", "code", "model", "training", "seed"), suffix=c(".full", ".baseline")) %>%
  select(resid.baseline, resid.full, code, surprisal.full, psychometric.full, model, training, seed, corpus, len.full) %>%
  mutate(resid.baseline.pol = if_else(resid.baseline > 0, 1, 0),
         resid.full.pol = if_else(resid.full > 0, 1, 0)) %>%
  mutate(resid.baseline = abs(resid.baseline),
         resid.full = abs(resid.full)) %>%
  mutate(resid_delta=resid.baseline - resid.full, #positive is better
         training_source=as.factor(str_replace(training, "-gptbpe", "")),
         bpe=str_detect(training, "gptbpe"))

r = resid_deltas %>%
  filter(resid.full.pol != resid.baseline.pol)
resid_deltas %>%
  ggplot(aes(x=surprisal.full, y=resid_delta, color=training)) +
    facet_grid(model~corpus) +
    geom_point(alpha=0.1, size=0.5)

language_model_data %>% filter(model == "gpt2")
resid_deltas %>%
  group_by(corpus) %>%
    mutate(psychometric = scale(psychometric.full)) %>%
  ungroup() %>%
  ggplot(aes(x=psychometric)) +
    theme_bw() +
    geom_density() +
    geom_vline(xintercept = 0, color = "grey") +
    facet_grid(.~corpus) +
    #coord_cartesian(xlim = c(-2, 4)) +
    theme(axis.title.x=element_blank(),
          axis.text.x=element_blank(),
          axis.ticks.x=element_blank())

#ggsave("length.png", width = 8, height = 1)

log_lik_deltas  %>%
#resid_deltas %>%
  #filter(resid.full.pol == resid.baseline.pol) %>%
  group_by(corpus) %>%
    mutate(psychometric = scale(psychometric)) %>%
  ungroup() %>%
  #filter(psychometric < 4) %>%
  #filter(len.full <= 10) %>%
  ggplot(aes(x = psychometric, y = delta_log_lik, color = model)) +
    theme_bw() +
    facet_grid(. ~ corpus, scales = "free") +
    #geom_rug(alpha = 0.003, sides = "b") +
    geom_hline(yintercept=0, color = "blue") +
    geom_vline(xintercept = 0, color = "grey") +
    geom_smooth(se = T, alpha = 0.2) +
    coord_cartesian(ylim = c(-0.1, 0.2), xlim = c(-2, 4)) +
    theme(legend.position = "bottom",
          strip.text.x = element_blank()) 
ggsave( "./resid_psycho.png", height = 4, width = 8)

NA
NA
resid_deltas %>%
  group_by(corpus) %>%
    mutate(psychometric = scale(psychometric.full)) %>%
  ungroup() %>%
  ggplot(aes(x=len.full)) +
    theme_bw() +
    geom_histogram(bins = 20) +
    geom_vline(xintercept = 0, color = "grey") +
    facet_grid(.~corpus) +
    coord_cartesian(xlim = c(1, 10)) +
    theme(axis.title.x=element_blank(),
          axis.text.x=element_blank(),
          axis.ticks.x=element_blank())
ggsave("length_maringals.png", width = 8, height = 1)


log_lik_deltas  %>%
#resid_deltas %>%
  #filter(resid.full.pol == resid.baseline.pol) %>%
  group_by(corpus) %>%
    mutate(psychometric = scale(psychometric)) %>%
  ungroup() %>%
  #filter(psychometric < 4) %>%
  #filter(len.full <= 10) %>%
  ggplot(aes(x = len, y = delta_log_lik, color = model)) +
    theme_bw() +
    facet_grid(. ~ corpus, scales = "free") +
    #geom_rug(alpha = 0.003, sides = "b") +
    geom_hline(yintercept=0, color = "blue") +
    geom_vline(xintercept = 0, color = "grey") +
    geom_smooth(se = T, alpha = 0.2) +
    coord_cartesian(ylim = c(-0.02, 0.06), xlim = c(1, 10)) +
    theme(legend.position = "bottom",
          strip.text.x = element_blank()) 
ggsave( "./resid_length.png", height = 4, width = 8)


word_norm = log_lik_deltas %>%
  drop_na() %>%
  group_by(word, corpus, model, training, seed) %>% 
  mutate(psychoword = scale(psychometric),
         norm_surp = scale(surprisal))
word_norm %>%
  ggplot(aes(x=norm_surp)) +
  facet_grid(~corpus) +
  geom_density() +
  coord_cartesian(xlim = c(-2, 5)) +
  geom_vline(xintercept = 0, color = "grey") +
  theme_bw() +
    theme(axis.title.x=element_blank(),
          axis.text.x=element_blank(),
          axis.ticks.x=element_blank())
ggsave("surp_maringals.png", width = 8, height = 1)

NA
NA

word_norm %>%
  ggplot(aes(x = psychoword, y = norm_surp, color = model)) +
    theme_bw() +
    facet_grid(. ~ corpus, scales = "free") +
    #geom_rug(alpha = 0.003, sides = "b") +
    geom_hline(yintercept=0, color = "blue") +
    geom_vline(xintercept = 0, color = "grey") +
    geom_smooth(se = T, alpha = 0.2) +
    #coord_cartesian(ylim = c(-0.05, 0.1), xlim = c(-2, 3)) +
    theme(legend.position = "bottom") 

#ggsave( "./resid_length.png", height = 4, width = 8)

word_norm %>%
  ggplot(aes(x = norm_surp, y = delta_log_lik, color = model)) +
    theme_bw() +
    facet_grid( . ~ corpus, scales = "free") +
    #geom_rug(alpha = 0.003, sides = "b") +
    geom_hline(yintercept=0, color = "blue") +
    geom_vline(xintercept = 0, color = "grey") +
    geom_smooth(se = T, alpha = 0.2) +
    coord_cartesian(ylim = c(-0.05, 0.07), xlim = c(-2, 5)) +
    theme(legend.position = "bottom") 
ggsave( "./norm_surp.png", height = 4, width = 8)


ngram_highsurp = word_norm %>%
  ungroup() %>%
  filter(corpus == "dundee", norm_surp > 2, model == "5gram") %>%
  select(code)

ngram_highsurp = ngram_highsurp$code

z = word_norm %>%
  ungroup() %>%
  filter(! code %in% ngram_highsurp) %>%
  filter(corpus == "dundee")

write.csv(z, "ngram-ablate.csv")
LS0tCnRpdGxlOiAiQ1VOWSAyMDIwIEFuYWx5c2lzIgpvdXRwdXQ6IGh0bWxfbm90ZWJvb2sKLS0tCgojIFBhY2thZ2VzIGFuZCB1dGlsaXRpZXMKCmBgYHtyfQpsaWJyYXJ5KHRpZHl2ZXJzZSkKbGlicmFyeShsbWU0KQpsaWJyYXJ5KGxtZXJUZXN0KQpsaWJyYXJ5KGxvZ2dpbmcpCmxpYnJhcnkobXZ0bm9ybSkKbGlicmFyeShtZ2N2KQojIFByb3ZpZGVzIGJvb3RzdHJhcCByZXNhbXBsaW5nIHRvb2xzCmxpYnJhcnkocnNhbXBsZSkKYGBgCgpgYGB7cn0KIyBDb21wdXRlIHRoZSBsb2ctbGlrZWxpaG9vZCBvZiBhIG5ldyBkYXRhc2V0IHVzaW5nIGEgZml0IGxtZTQgbW9kZWwuCmxvZ0xpa190ZXN0IDwtIGZ1bmN0aW9uKGxtLCB0ZXN0X1gsIHRlc3RfeSkgewogIHByZWRpY3Rpb25zIDwtIHByZWRpY3QobG0sIHRlc3RfWCwgcmUuZm9ybT1OQSkKICAjIEdldCBzdGQuZGV2LiBvZiByZXNpZHVhbCwgZXN0aW1hdGVkIGZyb20gdHJhaW4gZGF0YQogIHN0ZGV2IDwtIHNpZ21hKGxtKQogICMgRm9yIGVhY2ggcHJlZGljdGlvbi0tb2JzZXJ2YXRpb24sIGdldCB0aGUgZGVuc2l0eSBwKG9icyB8IE4ocHJlZGljdGVkLCBtb2RlbF9zaWdtYSkpIGFuZCByZWR1Y2UKICBkZW5zaXR5IDwtIHN1bShkbm9ybSh0ZXN0X3ksIHByZWRpY3Rpb25zLCBzdGRldiwgbG9nPVRSVUUpKQogIHJldHVybihkZW5zaXR5KQp9CiMgR2V0IHBlci1wcmVkaWN0aW9uIGxvZy1saWtlbGlob29kCmxvZ0xpa190ZXN0X3BlciA8LSBmdW5jdGlvbihsbSwgdGVzdF9YLCB0ZXN0X3kpIHsKICBwcmVkaWN0aW9ucyA8LSBwcmVkaWN0KGxtLCB0ZXN0X1gsIHJlLmZvcm09TkEpCiAgIyBHZXQgc3RkLmRldi4gb2YgcmVzaWR1YWwsIGVzdGltYXRlZCBmcm9tIHRyYWluIGRhdGEKICBzdGRldiA8LSBzaWdtYShsbSkKICAjIEZvciBlYWNoIHByZWRpY3Rpb24tLW9ic2VydmF0aW9uLCBnZXQgdGhlIGRlbnNpdHkgcChvYnMgfCBOKHByZWRpY3RlZCwgbW9kZWxfc2lnbWEpKQogIGRlbnNpdGllcyA8LSBkbm9ybSh0ZXN0X3ksIHByZWRpY3Rpb25zLCBzdGRldiwgbG9nPVRSVUUpCiAgcmV0dXJuKGRlbnNpdGllcykKfQojIENvbXB1dGUgTVNFIG9mIGEgbmV3IGRhdGFzZXQgdXNpbmcgYSBmaXQgbG1lNCBtb2RlbC4KbXNlX3Rlc3QgPC0gZnVuY3Rpb24obG0sIHRlc3RfWCwgdGVzdF95KSB7CiAgcmV0dXJuKG1lYW4oKHByZWRpY3QobG0sIHRlc3RfWCwgcmUuZm9ybT1OQSkgLSB0ZXN0X3kpIF4gMikpCn0KI1Nhbml0eSBjaGVja3MKI215bG0gPC0gZ2FtKHBzeWNob21ldHJpYyB+ICBzKHN1cnByaXNhbCwgYnMgPSAiY3IiLCBrID0gMjApICsgcyhwcmV2X3N1cnAsIGJzID0gImNyIiwgayA9IDIwKSArIHRlKGZyZXEsIGxlbiwgYnMgPSAiY3IiKSArIHRlKHByZXZfZnJlcSwgcHJldl9sZW4sIGJzID0gImNyIiksIGRhdGE9dHJhaW5fZGF0YSkKI2MobG9nTGlrKG15bG0pLCBsb2dMaWtfdGVzdChteWxtLCB0cmFpbl9kYXRhLCB0cmFpbl9kYXRhJHBzeWNob21ldHJpYykpCiNsb2dMaWtfdGVzdChteWxtLCB0ZXN0X2RhdGEsIHRlc3RfZGF0YSRwc3ljaG9tZXRyaWMpCmBgYAoKIyBEYXRhIGxvYWRpbmcgYW5kIHByZXByb2Nlc3NpbmcKCmBgYHtyIExvYWQgYW5kIHByZXByb2Nlc3MgZGF0YX0KZGF0YSA9IHJlYWQuY3N2KCIuLi9kYXRhL2hhcm1vbml6ZWRfcmVzdWx0cy5jc3YiKQoKYWxsX2RhdGEgPSBkYXRhICU+JQogIG11dGF0ZShzZWVkID0gYXMuZmFjdG9yKHNlZWQpKSAlPiUKICBncm91cF9ieShjb3JwdXMsIG1vZGVsLCB0cmFpbmluZywgc2VlZCkgJT4lCiAgICBtdXRhdGUocHJldl9zdXJwID0gbGFnKHN1cnByaXNhbCksCiAgICAgICAgIHByZXZfY29kZSA9IGxhZyhjb2RlKSwKICAgICAgICAgcHJldl9sZW4gPSBsYWcobGVuKSwKICAgICAgICAgcHJldl9mcmVxID0gbGFnKGZyZXEpLAogICAgICAgICBwcmV2X3N1cnAgPSBsYWcoc3VycHJpc2FsKSwKICAgICAgICAgCiAgICAgICAgIHByZXYyX2ZyZXEgPSBsYWcocHJldl9mcmVxKSwKICAgICAgICAgcHJldjJfY29kZSA9IGxhZyhwcmV2X2NvZGUpLAogICAgICAgICBwcmV2Ml9sZW4gPSBsYWcocHJldl9sZW4pLAogICAgICAgICBwcmV2Ml9zdXJwID0gbGFnKHByZXZfc3VycCksCiAgICAgICAgIAogICAgICAgICBwcmV2M19mcmVxID0gbGFnKHByZXYyX2ZyZXEpLAogICAgICAgICBwcmV2M19jb2RlID0gbGFnKHByZXYyX2NvZGUpLAogICAgICAgICBwcmV2M19sZW4gPSBsYWcocHJldjJfbGVuKSwKICAgICAgICAgcHJldjNfc3VycCA9IGxhZyhwcmV2Ml9zdXJwKSwKICAgICAgICAgCiAgICAgICAgIHByZXY0X2ZyZXEgPSBsYWcocHJldjNfZnJlcSksCiAgICAgICAgIHByZXY0X2NvZGUgPSBsYWcocHJldjNfY29kZSksCiAgICAgICAgIHByZXY0X2xlbiA9IGxhZyhwcmV2M19sZW4pLAogICAgICAgICBwcmV2NF9zdXJwID0gbGFnKHByZXYzX3N1cnApKSAlPiUKICB1bmdyb3VwKCkgJT4lCiAgCiAgIyBGaWx0ZXIgYmFjayB0d28gZm9yIHRoZSBkdW5kZWUgY29ycHVzLiBGaWx0ZXIgYmFjayAxIGZvciBhbGwgb3RoZXIgY29ycG9yYQogICMgTkIgdGhpcyBlZmZlY3RpdmVseSByZW1vdmVzIGFsbCB6ZXJvLXN1cnByaXNhbCByb3dzLCBzaW5jZSBlYXJseS1zZW50ZW5jZSB0b2tlbnMgZG9uJ3QgaGF2ZSBjb250aWd1b3VzIHRva2VuIGhpc3RvcnkKICBmaWx0ZXIoKGNvcnB1cyA9PSAiZHVuZGVlIiAmIGNvZGUgPT0gcHJldjJfY29kZSArIDIpIHwgKGNvcnB1cyAhPSAiZHVuZGVlIiAmIGNvZGUgPT0gcHJldjRfY29kZSArIDQpKSAlPiUKICAKICBzZWxlY3QoLXByZXZfY29kZSwgLXByZXYyX2NvZGUsIC1wcmV2M19jb2RlKSAlPiUKICBkcm9wX25hKCkKCmFsbF9kYXRhID0gYWxsX2RhdGEgJT4lCiAgbXV0YXRlKAogICAgbW9kZWwgPSBhcy5jaGFyYWN0ZXIobW9kZWwpLAogICAgbW9kZWwgPSBpZl9lbHNlKG1vZGVsID09ICJncHQtMiIsICJncHQyIiwgbW9kZWwpLAogICAgbW9kZWwgPSBhcy5mYWN0b3IobW9kZWwpKQpgYGAKCmBgYHtyfQptaXNzaW5nX3Jvd3MgPSBhbGxfZGF0YSAlPiUgY29tcGxldGUobmVzdGluZyhjb3JwdXMsIGNvZGUpLCBuZXN0aW5nKG1vZGVsLCB0cmFpbmluZywgc2VlZCkpICU+JSAKICBncm91cF9ieShjb3JwdXMsIGNvZGUpICU+JSAKICAgIGZpbHRlcihzdW0oaXMubmEoc3VycHJpc2FsKSkgPiAwKSAlPiUgCiAgdW5ncm91cCgpICU+JSAKICBhbnRpX2pvaW4oYWxsX2RhdGEsIGJ5PWMoImNvcnB1cyIsICJjb2RlIiwgIm1vZGVsIiwgInRyYWluaW5nIiwgInNlZWQiKSkKCm1pc3Npbmdfcm93cyAlPiUgZ2dwbG90KGFlcyh4PWNvcnB1cywgZmlsbD1mYWN0b3IocGFzdGUobW9kZWwsdHJhaW5pbmcpKSkpICsgZ2VvbV9iYXIocG9zaXRpb249cG9zaXRpb25fZG9kZ2Uod2lkdGg9MC44KSkKcHJpbnQobWlzc2luZ19yb3dzICU+JSBncm91cF9ieShtb2RlbCwgdHJhaW5pbmcsIHNlZWQsIGNvcnB1cykgJT4lIHN1bW1hcmlzZShuPW4oKSkpICU+JSBhcnJhbmdlKGRlc2MobikpCmBgYAoKCmBgYHtyIERyb3AgdG9rZW5zIGZvciB3aGljaCBhbnkgbW9kZWwgaXMgbWlzc2luZyBzdXJwcmlzYWwgZGF0YS59CgojIENvbXB1dGUgdGhlIGlkZWFsIG51bWJlciBvZiBtb2RlbC0tc2VlZC0tdHJhaW5pbmcgb2JzZXJ2YXRpb25zIHBlciB0b2tlbi4KdG9fZHJvcCA9IGFsbF9kYXRhICU+JQogIGdyb3VwX2J5KGNvcnB1cywgY29kZSkgJT4lIHN1bW1hcmlzZShuID0gbigpKSAlPiUgdW5ncm91cCgpICU+JQogIGdyb3VwX2J5KGNvcnB1cykgJT4lIG11dGF0ZSggbWF4X24gPSBtYXgobikpICU+JSB1bmdyb3VwKCkgJT4lCiAgZmlsdGVyKG1heF9uICE9IG4pICU+JQogIHNlbGVjdChjb2RlLCBjb3JwdXMpCgojdG9fZHJvcCA9IGFsbF9kYXRhICU+JSBncm91cF9ieShjb3JwdXMsIGNvZGUpICU+JSBmaWx0ZXIobigpICE9IGlkZWFsX3Rva2VuX29ic19jb3VudCkgJT4lIHVuZ3JvdXAoKQpsb2dpbmZvKHBhc3RlKCJEcm9wcGluZyIsIG5yb3codG9fZHJvcCksICJvYnNlcnZhdGlvbnMgY29ycmVzcG9uZGluZyB0byBjb3JwdXMgdG9rZW5zIHdoaWNoIGFyZSBtaXNzaW5nIG9ic2VydmF0aW9ucyBmb3Igc29tZSBtb2RlbC4iKSkKbG9naW5mbyhwYXN0ZSgiRHJvcHBpbmciLCB0b19kcm9wICU+JSBncm91cF9ieShjb3JwdXMsIGNvZGUpICU+JSBuX2dyb3VwcygpLCAidG9rZW5zIHdoaWNoIGFyZSBtaXNzaW5nIG9ic2VydmF0aW9ucyBmb3Igc29tZSBtb2RlbC4iKSkKCmFsbF9kYXRhID0gYWxsX2RhdGEgJT4lIGFudGlfam9pbih0b19kcm9wICU+JSBncm91cF9ieShjb3JwdXMsIGNvZGUpLCBieT1jKCJjb3JwdXMiLCAiY29kZSIpKQpsb2dpbmZvKHBhc3RlKCJBZnRlciBkcm9wLCIsIG5yb3coYWxsX2RhdGEpLCAib2JzZXJ2YXRpb25zICgiLCBhbGxfZGF0YSAlPiUgZ3JvdXBfYnkoY29ycHVzLCBjb2RlKSAlPiUgbl9ncm91cHMoKSwgIiB0b2tlbnMpIHJlbWFpbi4iKSkKYGBgCgpgYGB7ciBEcm9wIHRva2VucyBmb3Igd2hpY2ggYW55IG1vZGVsIGhhcyB6ZXJvLXZhbHVlZCBzdXJwcmlzYWxzLn0KCnRvX2Ryb3BfemVyb19zdXJwcyA9IGFsbF9kYXRhICU+JSBncm91cF9ieShjb3JwdXMsIGNvZGUpICU+JSBmaWx0ZXIoYW55KHN1cnByaXNhbCA9PSAwKSkgJT4lIHVuZ3JvdXAoKQpsb2dpbmZvKHBhc3RlKCJEcm9wcGluZyIsIG5yb3codG9fZHJvcF96ZXJvX3N1cnBzKSwgIm9ic2VydmF0aW9ucyBjb3JyZXNwb25kaW5nIHRvIGNvcnB1cyB0b2tlbnMgd2hpY2ggaGF2ZSBzdXJwcmlzYWwgemVyb3MgZm9yIHNvbWUgbW9kZWwuIikpCmxvZ2luZm8ocGFzdGUoIkRyb3BwaW5nIiwgdG9fZHJvcF96ZXJvX3N1cnBzICU+JSBncm91cF9ieShjb3JwdXMsIGNvZGUpICU+JSBuX2dyb3VwcygpLCAidG9rZW5zIHdoaWNoIGhhdmUgc3VycHJpc2FsIHplcm9zIGZvciBzb21lIG1vZGVsLiIpKQoKYWxsX2RhdGEgPSBhbGxfZGF0YSAlPiUgYW50aV9qb2luKHRvX2Ryb3BfemVyb19zdXJwcyAlPiUgZ3JvdXBfYnkoY29ycHVzLCBjb2RlKSwgYnk9YygiY29ycHVzIiwgImNvZGUiKSkKbG9naW5mbyhwYXN0ZSgiQWZ0ZXIgZHJvcCwiLCBucm93KGFsbF9kYXRhKSwgIm9ic2VydmF0aW9ucyAoIiwgYWxsX2RhdGEgJT4lIGdyb3VwX2J5KGNvcnB1cywgY29kZSkgJT4lIG5fZ3JvdXBzKCksICIgdG9rZW5zKSByZW1haW4uIikpCmBgYAoKYGBge3IgRHJvcCB0b2tlbnMgZm9yIHdoaWNoIHdlIGhhdmUgemVyby12YWx1ZWQgcHN5Y2hvbWV0cmljIGRhdGEufQoKdG9fZHJvcF96ZXJvX3BzeWNocyA9IGFsbF9kYXRhICU+JSBncm91cF9ieShjb3JwdXMsIGNvZGUpICU+JSBmaWx0ZXIoYW55KHBzeWNob21ldHJpYyA9PSAwKSkgJT4lIHVuZ3JvdXAoKQpsb2dpbmZvKHBhc3RlKCJEcm9wcGluZyIsIG5yb3codG9fZHJvcF96ZXJvX3BzeWNocyksICJvYnNlcnZhdGlvbnMgY29ycmVzcG9uZGluZyB0byBjb3JwdXMgdG9rZW5zIHdoaWNoIGhhdmUgcHN5Y2hvbWV0cmljIHplcm9zIGZvciBzb21lIG1vZGVsLiIpKQpsb2dpbmZvKHBhc3RlKCJEcm9wcGluZyIsIHRvX2Ryb3BfemVyb19wc3ljaHMgJT4lIGdyb3VwX2J5KGNvcnB1cywgY29kZSkgJT4lIG5fZ3JvdXBzKCksICJ0b2tlbnMgd2hpY2ggaGF2ZSBwc3ljaG9tZXRyaWMgemVyb3MgZm9yIHNvbWUgbW9kZWwuIikpCgphbGxfZGF0YSA9IGFsbF9kYXRhICU+JSBhbnRpX2pvaW4odG9fZHJvcF96ZXJvX3BzeWNocyAlPiUgZ3JvdXBfYnkoY29ycHVzLCBjb2RlKSwgYnk9YygiY29ycHVzIiwgImNvZGUiKSkKbG9naW5mbyhwYXN0ZSgiQWZ0ZXIgZHJvcCwiLCBucm93KGFsbF9kYXRhKSwgIm9ic2VydmF0aW9ucyAoIiwgYWxsX2RhdGEgJT4lIGdyb3VwX2J5KGNvcnB1cywgY29kZSkgJT4lIG5fZ3JvdXBzKCksICIgdG9rZW5zKSByZW1haW4uIikpCmBgYAoKIAojIExlYXJuIG1vZGVscwogCmBgYHtyfQojIENvbXB1dGUgbGluZWFyIG1vZGVsIHN0YXRzIGZvciB0aGUgZ2l2ZW4gdHJhaW5pbmcgZGF0YSBzdWJzZXQgYW5kIGZ1bGwgdGVzdCBkYXRhLgojIEF1dG9tYXRpY2FsbHkgc3Vic2V0cyB0aGUgdGVzdCBkYXRhIHRvIG1hdGNoIHRoZSByZWxldmFudCBncm91cCBmb3Igd2hpY2ggd2UgYXJlIHRyYWluaW5nIGEgbGluZWFyIG1vZGVsLgpnZXRfbG1fZGF0YSA8LSBmdW5jdGlvbihkZiwgdGVzdF9kYXRhLCBmb3JtdWxhLCBmb2xkLCBzdG9yZV9lbnYpIHsKICAjdGhpc19sbSA8LSBnYW0oZm9ybXVsYSwgZGF0YT1kZik7CiAgdGhpc19sbSA9IGxtKGZvcm11bGEsIGRhdGE9ZGYpCiAgdGhpc190ZXN0X2RhdGEgPC0gc2VtaV9qb2luKHRlc3RfZGF0YSwgZGYsIGJ5PWMoInRyYWluaW5nIiwgIm1vZGVsIiwgInNlZWQiLCAiY29ycHVzIikpOwogIAogICMgU2F2ZSBsbSB0byB0aGUgZ2xvYmFsIGVudiBzbyB0aGF0IHdlIGNhbiBhY2Nlc3MgcmVzaWR1YWxzIGxhdGVyLgogIGxtX25hbWUgPSBwYXN0ZSh1bmlxdWUocGFzdGUoZGYkbW9kZWwsIGRmJHRyYWluaW5nLCBkZiRzZWVkLCBkZiRjb3JwdXMpKVsxXSwgZm9sZCkKICBhc3NpZ24obG1fbmFtZSwgdGhpc19sbSwgZW52aXI9c3RvcmVfZW52KQogIAogIHN1bW1hcmlzZShkZiwKICAgICAgICAgICAgbG9nX2xpayA9IGFzLm51bWVyaWMobG9nTGlrKHRoaXNfbG0sIFJFTUwgPSBGKSksCiAgICAgICAgICAgIHRlc3RfbGlrID0gbG9nTGlrX3Rlc3QodGhpc19sbSwgdGhpc190ZXN0X2RhdGEsIHRoaXNfdGVzdF9kYXRhJHBzeWNob21ldHJpYyksCiAgICAgICAgICAgIHRlc3RfbXNlID0gbXNlX3Rlc3QodGhpc19sbSwgdGhpc190ZXN0X2RhdGEsIHRoaXNfdGVzdF9kYXRhJHBzeWNob21ldHJpYykpCn0KIyBGb3IgYSBwcmV2aW91c2x5IGZpdHRlZCBsbSBzdG9yZWQgaW4gc3RvcmVfZW52LCBnZXQgdGhlIHJlc2lkdWFscyBvbiB0ZXN0IGRhdGEgb2YgdGhlIHJlbGV2YW50IGRhdGEgc3Vic2V0LgpnZXRfbG1fcmVzaWR1YWxzIDwtIGZ1bmN0aW9uKGRmLCBmb2xkLCBzdG9yZV9lbnYpIHsKICAjIFJldHJpZXZlIHRoZSByZWxldmFudCBsbS4KICBsbV9uYW1lID0gcGFzdGUodW5pcXVlKHBhc3RlKGRmJG1vZGVsLCBkZiR0cmFpbmluZywgZGYkc2VlZCwgZGYkY29ycHVzKSlbMV0sIGZvbGQpCiAgdGhpc19sbSA8LSBnZXQobG1fbmFtZSwgZW52aXI9c3RvcmVfZW52KQogIAogIG11dGF0ZShkZiwKICAgICAgICAgbGlrZWxpaG9vZCA9IGxvZ0xpa190ZXN0X3Blcih0aGlzX2xtLCBkZiwgZGYkcHN5Y2hvbWV0cmljKSwKICAgICAgICAgcmVzaWQgPSBkZiRwc3ljaG9tZXRyaWMgLSBwcmVkaWN0KHRoaXNfbG0sIGRmLCByZS5mb3JtPU5BKSkKfQojIENvbXB1dGUgcGVyLWV4YW1wbGUgZGVsdGEtbG9nLWxpa2VsaWhvb2QgZm9yIHRoZSBnaXZlbiB0ZXN0IGZvbGQuCmdldF9sbV9kZWx0YV9sb2dfbGlrIDwtIGZ1bmN0aW9uKHRlc3RfZGF0YSwgZm9sZCwgYmFzZWxpbmVfZW52LCBmdWxsX2VudikgewogIGxtX25hbWUgPSBwYXN0ZSh1bmlxdWUocGFzdGUodGVzdF9kYXRhJG1vZGVsLCB0ZXN0X2RhdGEkdHJhaW5pbmcsIHRlc3RfZGF0YSRzZWVkLCB0ZXN0X2RhdGEkY29ycHVzKSlbMV0sIGZvbGQpCiAgYmFzZWxpbmVfbG0gPC0gZ2V0KGxtX25hbWUsIGVudmlyPWJhc2VsaW5lX2VudikKICBmdWxsX2xtIDwtIGdldChsbV9uYW1lLCBlbnZpcj1mdWxsX2VudikKICAKICBkZWx0YV9sb2dfbGlrID0gbG9nTGlrX3Rlc3RfcGVyKGZ1bGxfbG0sIHRlc3RfZGF0YSwgdGVzdF9kYXRhJHBzeWNob21ldHJpYykgLSBsb2dMaWtfdGVzdF9wZXIoYmFzZWxpbmVfbG0sIHRlc3RfZGF0YSwgdGVzdF9kYXRhJHBzeWNob21ldHJpYykKICByZXR1cm4oY2JpbmQodGVzdF9kYXRhLCBkZWx0YV9sb2dfbGlrPWRlbHRhX2xvZ19saWspKQp9CiMjIyMjCiMgRGVmaW5lIHJlZ3Jlc3Npb24gZm9ybXVsYWUuCiMgRXllLXRyYWNraW5nIHJlZ3Jlc3Npb246IG9ubHkgdXNlIHN1cnByaXNhbCBhbmQgcHJldmlvdXMgc3VycHJpc2FsOyBTUFJUIHJlZ3Jlc3Npb246IHVzZSAyLWJhY2sgZmVhdHVyZXMuCgojYmFzZWxpbmVfcnRfcmVncmVzc2lvbiA9IHBzeWNob21ldHJpYyB+IHRlKGZyZXEsIGxlbiwgYnMgPSAiY3IiKSArIHRlKHByZXZfZnJlcSwgcHJldl9sZW4sIGJzID0gImNyIikgKyB0ZShwcmV2Ml9mcmVxLCBwcmV2Ml9sZW4sIGJzID0gImNyIikKI2Jhc2VsaWVfc3BydF9yZWdyZXNzaW9uID0gcHN5Y2hvbWV0cmljIH4gdGUoZnJlcSwgbGVuLCBicyA9ICJjciIpICsgdGUocHJldl9mcmVxLCBwcmV2X2xlbiwgYnMgPSAiY3IiKSArIHRlKHByZXYyX2ZyZXEsIHByZXYyX2xlbiwgYnMgPSAiY3IiKSArIHRlKHByZXYzX2ZyZXEsIHByZXYzX2xlbiwgYnMgPSAiY3IiKSArIHRlKHByZXY0X2ZyZXEsIHByZXY0X2xlbiwgYnMgPSAiY3IiKQoKI2Z1bGxfcnRfcmVncmVzc2lvbiA9IHBzeWNob21ldHJpYyB+IHMoc3VycHJpc2FsLCBicyA9ICJjciIsIGsgPSAyMCkgKyBzKHByZXZfc3VycCwgYnMgPSAiY3IiLCBrID0gMjApICsgcyhwcmV2Ml9zdXJwLCBicyA9ICJjciIsIGsgPSAyMCkgKyB0ZShmcmVxLCBsZW4sIGJzID0gImNyIikgKyB0ZShwcmV2X2ZyZXEsIHByZXZfbGVuLCBicyA9ICJjciIpICsgdGUocHJldjJfZnJlcSwgcHJldjJfbGVuLCBicyA9ICJjciIpCiNmdWxsX3NwcnRfcmVncmVzc2lvbiA9IHBzeWNob21ldHJpYyB+IHMoc3VycHJpc2FsLCBicyA9ICJjciIsIGsgPSAyMCkgKyBzKHByZXZfc3VycCwgYnMgPSAiY3IiLCBrID0gMjApICsgcyhwcmV2Ml9zdXJwLCBicyA9ICJjciIsIGsgPSAyMCkgKyBzKHByZXYzX3N1cnAsIGJzID0gImNyIiwgayA9IDIwKSArIHMocHJldjRfc3VycCwgYnMgPSAiY3IiLCBrID0gMjApICsgdGUoZnJlcSwgbGVuLCBicyA9ICJjciIpICsgdGUocHJldl9mcmVxLCBwcmV2X2xlbiwgYnMgPSAiY3IiKSArIHRlKHByZXYyX2ZyZXEsIHByZXYyX2xlbiwgYnMgPSAiY3IiKSArIHRlKHByZXYzX2ZyZXEsIHByZXYzX2xlbiwgYnMgPSAiY3IiKSArIHRlKHByZXY0X2ZyZXEsIHByZXY0X2xlbiwgYnMgPSAiY3IiKQoKYmFzZWxpbmVfcnRfcmVncmVzc2lvbiA9IHBzeWNob21ldHJpYyB+IGZyZXEgKyBwcmV2X2ZyZXEgKyBwcmV2Ml9mcmVxICsgbGVuICsgcHJldl9sZW4gKyBwcmV2Ml9sZW4KYmFzZWxpbmVfc3BydF9yZWdyZXNzaW9uID0gcHN5Y2hvbWV0cmljIH4gZnJlcSArIHByZXZfZnJlcSArIHByZXYyX2ZyZXEgKyBwcmV2M19mcmVxICsgcHJldjRfZnJlcSArIGxlbiArIHByZXZfbGVuICsgcHJldjJfbGVuICsgcHJldjNfbGVuICsgcHJldjRfbGVuCgpmdWxsX3NwcnRfcmVncmVzc2lvbiA9IHBzeWNob21ldHJpYyB+IHN1cnByaXNhbCArIHByZXZfc3VycCArIHByZXYyX3N1cnAgKyBwcmV2M19zdXJwICsgcHJldjRfc3VycCArIGZyZXEgKyBwcmV2X2ZyZXEgKyBwcmV2Ml9mcmVxICsgcHJldjNfZnJlcSArIHByZXY0X2ZyZXEgKyBsZW4gKyBwcmV2X2xlbiArIHByZXYyX2xlbiArIHByZXYzX2xlbiArIHByZXY0X2xlbgpmdWxsX3J0X3JlZ3Jlc3Npb24gPSBwc3ljaG9tZXRyaWMgfiBzdXJwcmlzYWwgKyBwcmV2X3N1cnAgKyBwcmV2Ml9zdXJwICsgZnJlcSArIHByZXZfZnJlcSArIHByZXYyX2ZyZXEgKyBsZW4gKyBwcmV2X2xlbiArIHByZXYyX2xlbgogIAojIyMjIwojIFByZXBhcmUgZnJhbWVzL2Vudmlyb25tZW50cyBmb3Igc3RvcmluZyByZXN1bHRzL29iamVjdHMuCmJhc2VsaW5lX3Jlc3VsdHMgPSBkYXRhLmZyYW1lKCkKZnVsbF9tb2RlbF9yZXN1bHRzID0gZGF0YS5mcmFtZSgpCmJhc2VsaW5lX3Jlc2lkdWFscyA9IGRhdGEuZnJhbWUoKQpmdWxsX3Jlc2lkdWFscyA9IGRhdGEuZnJhbWUoKQpsb2dfbGlrX2RlbHRhcyA9IGRhdGEuZnJhbWUoKQoKI1JhbmRvbWx5IHNodWZmbGUgdGhlIGRhdGEKYWxsX2RhdGE8LWFsbF9kYXRhW3NhbXBsZShucm93KGFsbF9kYXRhKSksXQojQ3JlYXRlIEsgZXF1YWxseSBzaXplIGZvbGRzCksgPSAxMApmb2xkcyA8LSBjdXQoc2VxKDEsbnJvdyhhbGxfZGF0YSkpLGJyZWFrcz1LLGxhYmVscz1GQUxTRSkKI1BlcmZvcm0gMTAgZm9sZCBjcm9zcyB2YWxpZGF0aW9uCgojIEZpdCBtb2RlbHMgZm9yIHNvbWUgZm9sZCBvZiB0aGUgZGF0YS4KYmFzZWxpbmVfY29ycHVzID0gZnVuY3Rpb24oY29ycHVzLCBkZiwgdGVzdF9kYXRhLCBmb2xkLCBlbnYpIHsKICBpZihjb3JwdXMgPT0gImR1bmRlZSIpIHsKICAgIGdldF9sbV9kYXRhKGRmLCB0ZXN0X2RhdGEsIGJhc2VsaW5lX3J0X3JlZ3Jlc3Npb24sIGZvbGQsIGVudikKICB9IGVsc2UgewogICAgZ2V0X2xtX2RhdGEoZGYsIHRlc3RfZGF0YSwgYmFzZWxpbmVfc3BydF9yZWdyZXNzaW9uLCBmb2xkLCBlbnYpCiAgfQp9CmZ1bGxfbW9kZWxfY29ycHVzID0gZnVuY3Rpb24oY29ycHVzLCBkZiwgdGVzdF9kYXRhLCBmb2xkLCBlbnYpIHsKICBpZihjb3JwdXNbMV0gPT0gImR1bmRlZSIpIHsKICAgIGdldF9sbV9kYXRhKGRmLCB0ZXN0X2RhdGEsIGZ1bGxfcnRfcmVncmVzc2lvbiwgZm9sZCwgZW52KQogIH0gZWxzZSB7CiAgICBnZXRfbG1fZGF0YShkZiwgdGVzdF9kYXRhLCBmdWxsX3NwcnRfcmVncmVzc2lvbiwgZm9sZCwgZW52KQogIH0KfQoKIyBQcmVwYXJlIGEgbmV3IEVudmlyb25tZW50IGluIHdoaWNoIHdlIHN0b3JlIGZpdHRlZCBMTXMsIHdoaWNoIHdlJ2xsIHF1ZXJ5IGxhdGVyIGZvciByZXNpZHVhbHMgYW5kIG90aGVyIG1ldHJpY3MuCmJhc2VsaW5lX2VudiA9IG5ldy5lbnYoKQpmdWxsX2VudiA9IG5ldy5lbnYoKQoKZm9yKGkgaW4gMTpLKSB7IAogICNTZWdlbWVudCB5b3VyIGRhdGEgYnkgZm9sZCB1c2luZyB0aGUgd2hpY2goKSBmdW5jdGlvbiAKICB0ZXN0SW5kZXhlcyA8LSB3aGljaChmb2xkcz09aSwgYXJyLmluZD1UUlVFKQogIHRlc3RfZGF0YSA8LSBhbGxfZGF0YVt0ZXN0SW5kZXhlcywgXQogIHRyYWluX2RhdGEgPC0gYWxsX2RhdGFbLXRlc3RJbmRleGVzLCBdCiAgCiAgIyBDb21wdXRlIGEgYmFzZWxpbmUgbGluZWFyIG1vZGVsIGZvciBlYWNoIG1vZGVsLS10cmFpbmluZy0tc2VlZC0tUlQtY29ycHVzIGNvbWJpbmF0aW9uLgogIGJhc2VsaW5lcyA9IHRyYWluX2RhdGEgJT4lCiAgICBncm91cF9ieShtb2RlbCwgdHJhaW5pbmcsIHNlZWQsIGNvcnB1cykgJT4lCiAgICAgIHByaW50KG1vZGVsKSAlPiUKICAgICAgZG8oYmFzZWxpbmVfY29ycHVzKHVuaXF1ZSguJGNvcnB1cyksIC4sIHRlc3RfZGF0YSwgaSwgYmFzZWxpbmVfZW52KSkgJT4lCiAgICB1bmdyb3VwKCkgJT4lCiAgICBtdXRhdGUoc2VlZCA9IGFzLmZhY3RvcihzZWVkKSwKICAgICAgICAgICBmb2xkID0gaSkKICAKICBiYXNlbGluZV9yZXN1bHRzID0gcmJpbmQoYmFzZWxpbmVfcmVzdWx0cywgYmFzZWxpbmVzKQogIAogICMgQ29tcHV0ZSBhIGZ1bGwgbGluZWFyIG1vZGVsIGZvciBlYWNoIG1vZGVsLS10cmFpbmluZy0tc2VlZC1SVC1jb3JwdXMgY29tYmluYXRpb24KICBmdWxsX21vZGVscyA9IHRyYWluX2RhdGEgJT4lCiAgICBncm91cF9ieShtb2RlbCwgdHJhaW5pbmcsIHNlZWQsIGNvcnB1cykgJT4lCiAgICAgIGRvKGZ1bGxfbW9kZWxfY29ycHVzKHVuaXF1ZSguJGNvcnB1cyksIC4sIHRlc3RfZGF0YSwgaSwgZnVsbF9lbnYpKSAlPiUKICAgIHVuZ3JvdXAoKSAlPiUKICAgIG11dGF0ZShzZWVkID0gYXMuZmFjdG9yKHNlZWQpLAogICAgICAgICAgIGZvbGQgPSBpKQogIAogIGZ1bGxfbW9kZWxfcmVzdWx0cyA9IHJiaW5kKGZ1bGxfbW9kZWxfcmVzdWx0cywgZnVsbF9tb2RlbHMpCiAgCiAgIyBDb21wdXRlIGRlbHRhLWxvZy1saWtlbGlob29kcwogIGZvbGRfbG9nX2xpa19kZWx0YXMgPSB0ZXN0X2RhdGEgJT4lCiAgICBncm91cF9ieShtb2RlbCwgdHJhaW5pbmcsIHNlZWQsIGNvcnB1cykgJT4lCiAgICAgIGRvKGdldF9sbV9kZWx0YV9sb2dfbGlrKC4sIGksIGJhc2VsaW5lX2VudiwgZnVsbF9lbnYpKSAlPiUKICAgIHVuZ3JvdXAoKQoKICBsb2dfbGlrX2RlbHRhcyA9IHJiaW5kKGxvZ19saWtfZGVsdGFzLCBmb2xkX2xvZ19saWtfZGVsdGFzKQogIAogIGZvbGRfYmFzZWxpbmVfcmVzaWR1YWxzID0gdGVzdF9kYXRhICU+JQogICAgZ3JvdXBfYnkobW9kZWwsIHRyYWluaW5nLCBzZWVkLCBjb3JwdXMpICU+JQogICAgICBkbyhnZXRfbG1fcmVzaWR1YWxzKC4sIGksIGJhc2VsaW5lX2VudikpICU+JQogICAgdW5ncm91cCgpCgogIGJhc2VsaW5lX3Jlc2lkdWFscyA9IHJiaW5kKGJhc2VsaW5lX3Jlc2lkdWFscywgZm9sZF9iYXNlbGluZV9yZXNpZHVhbHMpCgogIGZvbGRfZnVsbF9yZXNpZHVhbHMgPSB0ZXN0X2RhdGEgJT4lCiAgICBncm91cF9ieShtb2RlbCwgdHJhaW5pbmcsIHNlZWQsIGNvcnB1cykgJT4lCiAgICAgIGRvKGdldF9sbV9yZXNpZHVhbHMoLiwgaSwgZnVsbF9lbnYpKSAlPiUKICAgIHVuZ3JvdXAoKQoKICBmdWxsX3Jlc2lkdWFscyA9IHJiaW5kKGZ1bGxfcmVzaWR1YWxzLCBmb2xkX2Z1bGxfcmVzaWR1YWxzKQp9CmBgYAoKYGBge3J9CiN3cml0ZS5jc3YoZnVsbF9yZXNpZHVhbHMsICIuLi9kYXRhL2FuYWx5c2lzX2NoZWNrcG9pbnRzL2Z1bGxfcmVzaWR1YWxzLmNzdiIpCiN3cml0ZS5jc3YoYmFzZWxpbmVfcmVzaWR1YWxzLCAiLi4vZGF0YS9hbmFseXNpc19jaGVja3BvaW50cy9iYXNlbGluZV9yZXNpZHVhbHMuY3N2IikKYGBgCgpgYGB7cn0KbW9kZWxfZGVsdGFzID0gbG9nX2xpa19kZWx0YXMgJT4lCiAgZ3JvdXBfYnkobW9kZWwsIHRyYWluaW5nLCBzZWVkLCBjb3JwdXMpICU+JSAKICBzdW1tYXJpc2UobWVhbl9kZWx0YV9sb2dfbGlrID0gbWVhbihkZWx0YV9sb2dfbGlrKSwKICAgICAgICAgICAgc2VtX2RlbHRhX2xvZ19saWsgPSBzZChkZWx0YV9sb2dfbGlrKSAvIHNxcnQobGVuZ3RoKGRlbHRhX2xvZ19saWspKSkKYGBgCgpgYGB7cn0Kd3JpdGUuY3N2KGZ1bGxfbW9kZWxfcmVzdWx0cywgIi4uL2RhdGEvYW5hbHlzaXNfY2hlY2twb2ludHMvZnVsbF9tb2RlbF9yZXN1bHQuY3N2IikKd3JpdGUuY3N2KGJhc2VsaW5lX3Jlc3VsdHMsICIuLi9kYXRhL2FuYWx5c2lzX2NoZWNrcG9pbnRzL2Jhc2VsaW5lX3Jlc3VsdHMuY3N2IikKI2Z1bGxfbW9kZWxfcmVzdWx0cyA9IHJlYWQuY3N2KCIuLi9kYXRhL2FuYWx5c2lzX2NoZWNrcG9pbnRzL2ZmdWxsX21vZGVsX3Jlc3VsdHMuY3N2IikKI2Jhc2VsaW5lX3Jlc3VsdHMgPSByZWFkLmNzdigiLi4vZGF0YS9hbmFseXNpc19jaGVja3BvaW50cy9mYmFzZWxpbmVfcmVzdWx0c2IuY3N2IikKYGBgCgpgYGB7cn0KbWV0cmljIDwtICLOlExvZ0xpayIKI21ldHJpYyA8LSAiLc6UTVNFIgoKIyAjIFNlbGVjdCB0aGUgcmVsZXZhbnQgbWV0cmljLgptb2RlbF9kZWx0YXMgPSBtb2RlbF9kZWx0YXMgJT4lCiAgICAjIFJldHJpZXZlIHRoZSBjdXJyZW50IHRlc3QgbWV0cmljCiAgICBtdXRhdGUoZGVsdGFfdGVzdF9tZWFuID0gbWVhbl9kZWx0YV9sb2dfbGlrLAogICAgICAgICAgIGRlbHRhX3Rlc3Rfc2VtID0gc2VtX2RlbHRhX2xvZ19saWspICU+JQogICAgIyBtdXRhdGUoZGVsdGFfdGVzdF9tZWFuID0gbWVhbl9kZWx0YV9tc2UsCiAgICAjICAgICAgICBkZWx0YV90ZXN0X3NlbSA9IHNlbV9kZWx0YV9tc2UpCiAgICAKICAgICMgUmVtb3ZlIHRoZSByYXcgbWV0cmljcy4KICAgIHNlbGVjdCgtbWVhbl9kZWx0YV9sb2dfbGlrLCAtc2VtX2RlbHRhX2xvZ19saWssCiAgICAgICAgICAgIy1tZWFuX2RlbHRhX21zZSwgLXNlbV9kZWx0YV9tc2UKICAgICAgICAgICApCm1vZGVsX2RlbHRhcwpgYGAKCmBgYHtyfQojIFNhbml0eSBjaGVjazogdHJhaW5pbmcgb24gdHJhaW4rdGVzdCBkYXRhIHNob3VsZCB5aWVsZCBpbXByb3ZlZCBwZXJmb3JtYW5jZSBvdmVyIHRyYWluaW5nIG9uIGp1c3QgdHJhaW5pbmcgZGF0YS4gKFdoZW4gZXZhbHVhdGluZyBvbiB0ZXN0IGRhdGEuKQojIGZ1bGxfYmFzZWxpbmVzID0gYWxsX2RhdGEgJT4lCiMgICBncm91cF9ieShtb2RlbCwgdHJhaW5pbmcsIHNlZWQsIGNvcnB1cykgJT4lCiMgICBzdW1tYXJpc2UoYmFzZWxpbmVfdHJhaW5fYWxsX3Rlc3RfbGlrID0gbG9nTGlrX3Rlc3QobG0ocHN5Y2hvbWV0cmljIH4gbGVuICsgZnJlcSArIHNlbnRfcG9zLCBkYXRhPS4pLCBzZW1pX2pvaW4odGVzdF9kYXRhLCAuLCBieT1jKCJ0cmFpbmluZyIsICJtb2RlbCIsICJzZWVkIiwgImNvcnB1cyIpKSwgc2VtaV9qb2luKHRlc3RfZGF0YSwgLiwgYnk9YygidHJhaW5pbmciLCAibW9kZWwiLCAic2VlZCIsICJjb3JwdXMiKSkkcHN5Y2hvbWV0cmljKSkgJT4lCiMgICB1bmdyb3VwKCkKIyBmdWxsX2Jhc2VsaW5lcwojIAojIGZ1bGxfYmFzZWxpbmVzICU+JQojICAgcmlnaHRfam9pbihiYXNlbGluZXMsIGJ5PWMoInNlZWQiLCAidHJhaW5pbmciLCAibW9kZWwiLCAiY29ycHVzIikpICU+JQojICAgbXV0YXRlKGRlbHRhPWJhc2VsaW5lX3RyYWluX2FsbF90ZXN0X2xpay1iYXNlbGluZV90ZXN0X2xpaykgJT4lCiMgICBzZWxlY3QoLWJhc2VsaW5lX2xpaykgIyAlPiUKIyAgICNzZWxlY3QoLWJhc2VsaW5lX3Rlc3RfbGlrLCAtYmFzZWxpbmVfdHJhaW5fYWxsX3Rlc3RfbGlrLCAtYmFzZWxpbmVfbGlrLCAtYmFzZWxpbmVfdGVzdF9tc2UpCmBgYAoKIyBMb2FkIGxhbmd1YWdlIG1vZGVsIGRhdGEgKFN5bnRheEd5bSwgUFBMKQoKYGBge3J9Cmxhbmd1YWdlX21vZGVsX2RhdGEgPSByZWFkLmNzdigiLi4vZGF0YS9tb2RlbF9tZXRhZGF0YS5jc3YiKSAlPiUKICBtdXRhdGUobW9kZWwgPSBhcy5jaGFyYWN0ZXIobW9kZWwpLAogICAgICAgICBtb2RlbCA9IGlmX2Vsc2UobW9kZWwgPT0gImdwdC0yIiwgImdwdDIiLCBtb2RlbCksCiAgICAgICAgIG1vZGVsID0gYXMuZmFjdG9yKG1vZGVsKSkgJT4lCiAgbXV0YXRlKHRyYWluX3NpemUgPSBjYXNlX3doZW4oc3RyX3N0YXJ0cyh0cmFpbmluZywgImJsbGlwLWxnIikgfiA0MiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBzdHJfc3RhcnRzKHRyYWluaW5nLCAiYmxsaXAtbWQiKSB+IDE1LAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHN0cl9zdGFydHModHJhaW5pbmcsICJibGxpcC1zbSIpIH4gNSwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBzdHJfc3RhcnRzKHRyYWluaW5nLCAiYmxsaXAteHMiKSB+IDEpLAogICAgICAgICAKICAgICAgICAgIyBUcmFpbmluZyB2b2NhYnVsYXJ5IHVzdWFsbHkgY292YXJpZXMgd2l0aCB0aGUgdHJhaW5pbmcgY29ycHVzLgogICAgICAgICAjIEJ1dCBCUEUgbW9kZWxzIHNoYXJlIGEgdm9jYWJ1bGFyeSBhY3Jvc3MgdHJhaW5pbmcgY29ycG9yYS4KICAgICAgICAgdHJhaW5pbmdfdm9jYWI9YXMuZmFjdG9yKGlmZWxzZShzdHJfZGV0ZWN0KHRyYWluaW5nLCAiZ3B0YnBlIiksICJncHRicGUiLCBhcy5jaGFyYWN0ZXIodHJhaW5pbmcpKSksCiAgICAgICAgIHRyYWluaW5nX3NvdXJjZT1hcy5mYWN0b3Ioc3RyX3JlcGxhY2UoYXMuY2hhcmFjdGVyKHRyYWluaW5nKSwgIi1ncHRicGUiLCAiIikpCiAgICAgICAgICkgJT4lCiAgbXV0YXRlKHNlZWQgPSBhcy5mYWN0b3Ioc2VlZCkpICU+JQogIHNlbGVjdCgtcGlkLCAtdGVzdF9sb3NzKSAlPiUKICBkaXN0aW5jdChtb2RlbCwgdHJhaW5pbmcsIHNlZWQsIC5rZWVwX2FsbCA9IFRSVUUpCnRhYmxlKGxhbmd1YWdlX21vZGVsX2RhdGEkc2VlZCkKdGFibGUobW9kZWxfZGVsdGFzJHNlZWQpCmBgYAoKRmlyc3Qgam9pbiBkZWx0YS1tZXRyaWMgZGF0YSB3aXRoIG1vZGVsIGF1eGlsaWFyeSBkYXRhLgoKYGBge3J9Cm1vZGVsX2RlbHRhcyA9IG1vZGVsX2RlbHRhcyAlPiUKICBtZXJnZShsYW5ndWFnZV9tb2RlbF9kYXRhLCBieSA9IGMoInNlZWQiLCAidHJhaW5pbmciLCAibW9kZWwiKSwgYWxsPVQpICU+JQogIGRyb3BfbmEoKQoKbW9kZWxfZGVsdGFzCmBgYAoKQWxzbyBqb2luIG9uIHRoZSBvcmlnaW5hbCBsaW5lYXIgbW9kZWwgZGF0YSwgcmF0aGVyIHRoYW4gY29sbGFwc2luZyB0byBkZWx0YS1tZXRyaWNzLgpUaGlzIHdpbGwgc3VwcG9ydCByZWdyZXNzaW9ucyBsYXRlciBvbiB0aGF0IGRvbid0IGNvbGxhcHNlIGFjcm9zcyBmb2xkcy4KCgojIEZpbmFsIGRhdGEgcHJlcHJvY2Vzc2luZwoKYGBge3IgRmlsdGVyIG1vZGVscyBhbmQvb3IgY29ycG9yYX0KIyBFeGNsdWRlIG9yZGVyZWQtbmV1cm9ucyBmcm9tIGFsbCBhbmFseXNlcy4KbW9kZWxfZGVsdGFzIDwtIG1vZGVsX2RlbHRhcyAlPiUKICBmaWx0ZXIobW9kZWwgIT0gIm9yZGVyZWQtbmV1cm9ucyIpCmBgYAoKCiMgVmlzdWFsaXphdGlvbnMKCiMjIFRoZSBiYXNpY3MKCmBgYHtyLCBmaWcuY2FwPSJDb3JwdXMgc2l6ZXMifQphbGxfZGF0YSAlPiUgZ2dwbG90KGFlcyh4PWNvcnB1cykpICsgZ2VvbV9iYXIoKQpwcmludChhbGxfZGF0YSAlPiUgZ3JvdXBfYnkoY29ycHVzKSAlPiUgc3VtbWFyaXNlKG49bigpKSkKYGBgCgoKYGBge3IsIGZpZy5jYXA9IldvcmQgZnJlcXVlbmN5IGRpc3RyaWJ1dGlvbiBieSBjb3JwdXMifQphbGxfZGF0YSAlPiUgCiAgZ2dwbG90KGFlcyh4PWZyZXEsIGNvbG9yPWNvcnB1cykpICsgZ2VvbV9kZW5zaXR5KCkKYGBgCgpgYGB7ciwgZmlnLmNhcD0iV29yZCBsZW5ndGggZGlzdHJpYnV0aW9uIGJ5IGNvcnB1cyJ9CmFsbF9kYXRhICU+JSAKICBnZ3Bsb3QoYWVzKHg9bGVuLCBjb2xvcj1jb3JwdXMpKSArIGdlb21fZGVuc2l0eSgpCmBgYAoKYGBge3IsIGZpZy5jYXA9IlN1cnByaXNhbCBkaXN0cmlidXRpb24gYnkgY29ycHVzIn0KYWxsX2RhdGEgJT4lIAogIGdncGxvdChhZXMoeD1zdXJwcmlzYWwsIGNvbG9yPWNvcnB1cykpICsgZ2VvbV9kZW5zaXR5KCkKYGBgCgojIyBQcmVkaWN0aXZlIHBvd2VyIGFuZCBTRwoKCmBgYHtyIEJ5IG1vZGVsfQptb2RlbF9kZWx0YXMgJT4lCiAgZ2dwbG90KGFlcyh4PXNnX3Njb3JlLCB5PWRlbHRhX3Rlc3RfbWVhbikpICsKICAgIGdlb21fZXJyb3JiYXIoYWVzKHltaW49ZGVsdGFfdGVzdF9tZWFuLWRlbHRhX3Rlc3Rfc2VtLCB5bWF4PWRlbHRhX3Rlc3RfbWVhbitkZWx0YV90ZXN0X3NlbSkpICsKICAgIGdlb21fc21vb3RoKG1ldGhvZD0ibG0iLCBzZT1UKSArCiAgICBnZW9tX3BvaW50KHN0YXQ9ImlkZW50aXR5IiwgcG9zaXRpb249ImRvZGdlIiwgYWxwaGE9MSwgc2l6ZT0zLCBhZXMoY29sb3I9dHJhaW5pbmdfdm9jYWIsIHNoYXBlPW1vZGVsKSkgKwogICAgeWxhYihtZXRyaWMpICsKICAgIHhsYWIoIlN5bnRheCBHZW5lcmFsaXphdGlvbiBTY29yZSIpICsKICAgIGdndGl0bGUoIlN5bnRhY3RpYyBHZW5lcmFsaXphdGlvbiB2cy4gUHJlZGljdGl2ZSBQb3dlciIpICsKICAgIHNjYWxlX2NvbG9yX21hbnVhbCh2YWx1ZXMgPSBjKCJibGxpcC1sZyI9IiM0NDAxNTRGRiIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJibGxpcC1tZCI9IiMzOTU2OENGRiIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJibGxpcC1zbSI9IiMxRjk2OEJGRiIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJibGxpcC14cyI9IiM3M0QwNTVGRiIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJncHRicGUiPSIjODg4ODg4IikpICsKICAgIGZhY2V0X2dyaWQofmNvcnB1cywgc2NhbGVzPSJmcmVlIikgKwogICAgdGhlbWUoYXhpcy50ZXh0PWVsZW1lbnRfdGV4dChzaXplPTE0KSwKICAgICAgICAgIHN0cmlwLnRleHQueCA9IGVsZW1lbnRfdGV4dChzaXplPTE0KSwKICAgICAgICAgIGxlZ2VuZC50ZXh0PWVsZW1lbnRfdGV4dChzaXplPTE0KSwKICAgICAgICAgIGF4aXMudGl0bGU9ZWxlbWVudF90ZXh0KHNpemU9MTgpLAogICAgICAgICAgbGVnZW5kLnBvc2l0aW9uID0gImJvdHRvbSIpCiNnZ3NhdmUoIi4vY29nc2NpX2ltYWdlcy9zZ19sb2dsaWsucG5nIixoZWlnaHQ9NSx3aWR0aD02KQpgYGAKCiMjIyBSZWdyZXNzaW9uIGFuYWx5c2VzCgpXZSBjb250cm9sIGZvciBlZmZlY3RzIG9mIHBlcnBsZXhpdHkgYnkgcmVsYXRpbmcgdGhlIHJlc2lkdWFscyBvZiBhIGBwZXJmb3JtYW5jZSB+IFBQTGAgcmVncmVzc2lvbiB0byBTRyBzY29yZS4KCmBgYHtyIFJlc2lkdWFsaXplZCByZWdyZXNzaW9ufQojIFByZXBhcmUgYSByZXNpZHVhbGl6ZWQgcmVncmVzc2lvbiBmb3IgeDEgb250byB5LCBjb250cm9sbGluZyBmb3IgdGhlIGVmZmVjdHMgb2YgeDIuCmRfcmVzaWQgPSBtb2RlbF9kZWx0YXMgJT4lCiAgZHJvcF9uYSgpICU+JQogIAogIGdyb3VwX2J5KGNvcnB1cykgJT4lCiAgICAjIFJlc2lkdWFsaXplIGRlbHRhIG1ldHJpYyB3LnIudCBQUEwgZm9yIGVhY2ggbW9kZWwtLXRyYWluaW5nLS1zZWVkIHdpdGhpbgogICAgIyB0cmFpbmluZyB2b2NhYnVsYXJ5CiAgICBtdXRhdGUocmVzaWQuZGVsdGEgPSByZXNpZChsbShkZWx0YV90ZXN0X21lYW4gfiB0cmFpbmluZ192b2NhYjp0ZXN0X3BwbCkpKSAlPiUKICAgICMgUmVzaWR1YWxpemUgU0cgc2NvcmUgdy5yLnQuIFBQTCBmb3IgZWFjaCBtb2RlbC0tdHJhaW5pbmctLXNlZWQKICAgICMgd2l0aGluIHRyYWluaW5nIHZvY2FidWxhcnkKICAgIG11dGF0ZShyZXNpZC5zZyA9IHJlc2lkKGxtKHNnX3Njb3JlIH4gdHJhaW5pbmdfdm9jYWI6dGVzdF9wcGwpKSkgJT4lCiAgdW5ncm91cCgpCgojIE5vdyBwbG90IHJlc2lkdWFsIHZzIFNHCmRfcmVzaWQgJT4lCiAgI2ZpbHRlcihjb3JwdXMgIT0gImJuYy1icm93biIpICU+JQogIGdncGxvdChhZXMoeD1yZXNpZC5zZywgeT1yZXNpZC5kZWx0YSkpICsKICAgIHRoZW1lX2J3KCkgKwogICAgc2NhbGVfc2hhcGVfbWFudWFsKHZhbHVlcyA9IGMoMTYsIDE3LCAxNSwgMTgpKSArCiAgICBnZW9tX3Ntb290aChtZXRob2Q9ImxtIiwgc2U9VCwgYWxwaGE9MC4zKSArCiAgICBnZW9tX3BvaW50KHN0YXQ9ImlkZW50aXR5IiwgcG9zaXRpb249ImRvZGdlIiwgYWxwaGE9MSwgc2l6ZT00LCBhZXMoc2hhcGU9bW9kZWwsIGNvbG9yPXRyYWluaW5nX3ZvY2FiKSkgKwogICAgeWxhYihwYXN0ZSgiUmVzaWR1YWwiLCBtZXRyaWMpKSArCiAgICB4bGFiKCJSZXNpZHVhbCBTeW50YXggR2VuZXJhbGl6YXRpb24gU2NvcmUiKSArCiAgICBnZ3RpdGxlKCJTeW50YWN0aWMgR2VuZXJhbGl6YXRpb24gdnMuIFByZWRpY3RpdmUgUG93ZXIiKSArCiAgICBzY2FsZV9jb2xvcl9tYW51YWwodmFsdWVzID0gYygiYmxsaXAtbGciPSIjNDQwMTU0RkYiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImJsbGlwLW1kIj0iIzM5NTY4Q0ZGIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJibGxpcC1zbSI9IiMxRjk2OEJGRiIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiYmxsaXAteHMiPSIjNzNEMDU1RkYiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImdwdGJwZSI9IiNmMDk0MWYiKSkgKwogICAgZmFjZXRfZ3JpZCgufmNvcnB1cywgc2NhbGVzPSJmcmVlIikgKwogICAgdGhlbWUoYXhpcy50ZXh0PWVsZW1lbnRfdGV4dChzaXplPTE0KSwKICAgICAgICAgIHN0cmlwLnRleHQueCA9IGVsZW1lbnRfdGV4dChzaXplPTE0KSwKICAgICAgICAgIGxlZ2VuZC50ZXh0PWVsZW1lbnRfdGV4dChzaXplPTE0KSwKICAgICAgICAgIGF4aXMudGl0bGU9ZWxlbWVudF90ZXh0KHNpemU9MTgpLAogICAgICAgICAgbGVnZW5kLnBvc2l0aW9uID0gInJpZ2h0IikKZ2dzYXZlKCIuLi9pbWFnZXMvY3VueTIwMjAvZGxsX3NnLnBuZyIsaGVpZ2h0PTQuNSx3aWR0aD05KQpgYGAKCgpgYGB7ciBTdGVwd2lzZSByZWdyZXNzaW9ufQpkb19zdGVwd2lzZV9yZWdyZXNzaW9uID0gZnVuY3Rpb24oY3VyX2NvcnB1cykgewogIHJlZ3Jlc3Npb25fZGF0YSA9IG1vZGVsX2RlbHRhcyAlPiUKICAgIGZpbHRlcihjb3JwdXMgPT0gY3VyX2NvcnB1cykKICAKICBwcmludCgiLS0tLS0tLS0tLS0tLS0tLS0tLS0tLSIpCiAgcHJpbnQoY3VyX2NvcnB1cykKICAKICBsbTEgPSBsbShkZWx0YV90ZXN0X21lYW4gfiB0cmFpbmluZ192b2NhYjp0ZXN0X3BwbCwgZGF0YSA9IHJlZ3Jlc3Npb25fZGF0YSkKICBsbTIgPSBsbShkZWx0YV90ZXN0X21lYW4gfiB0cmFpbmluZ192b2NhYjp0ZXN0X3BwbCArIHNnX3Njb3JlLCBkYXRhID0gcmVncmVzc2lvbl9kYXRhKQogIHByaW50KGFub3ZhKGxtMSwgbG0yKSkKICBzdW1tYXJ5KGxtMikKfQpkb19zdGVwd2lzZV9yZWdyZXNzaW9uKCJibmMtYnJvd24iKQpkb19zdGVwd2lzZV9yZWdyZXNzaW9uKCJkdW5kZWUiKQpkb19zdGVwd2lzZV9yZWdyZXNzaW9uKCJuYXR1cmFsLXN0b3JpZXMiKQpgYGAKCmBgYHtyIFNhbml0eSBjaGVjazogZXF1aXZhbGVuY2UgYmV0d2VlbiBhbmFseXNlc30KIyBUaGUgcmVzaWR1YWxpemVkIGFuYWx5c2lzIGFuZCB0aGUgc3RlcHdpc2UgcmVncmVzc2lvbiBhbmFseXNpcwojIHNob3VsZCB5aWVsZCB0aGUgc2FtZSBjb2VmZmljaWVudHMgZm9yIHRoZSBTRyBzY29yZSB2YXJpYWJsZS4KIwojIEJlbG93LCB3ZSBjb21wdXRlIHRoZSBzbG9wZSBjb2VmZmljaWVudCBmb3IgdGhlIFNHIHRlcm0gaW4gdGhlCiMgcmVzaWR1YWxpemVkIGFuYWx5c2VzLgojCiMgVGhlc2UgY29lZmZpY2llbnRzIHNob3VsZCBtYXRjaCB0aG9zZSBmb3VuZCBpbiB0aGUgc3RlcHdpc2UKIyByZWdyZXNzaW9uIGZvciBgc2dfc2NvcmVgIGFib3ZlLgpkX3Jlc2lkICU+JSBncm91cF9ieShjb3JwdXMpICU+JQogIGdyb3VwX21vZGlmeSh+dGlkeShsbShyZXNpZC5kZWx0YSB+IHRyYWluaW5nX3ZvY2FiOnRlc3RfcHBsICsgcmVzaWQuc2csIGRhdGE9LikpCiAgICAgICAgICAgICAgICAgJT4lIGZpbHRlcih0ZXJtID09ICJyZXNpZC5zZyIpKSAlPiUgCiAgc2VsZWN0KGNvcnB1cywgZXN0aW1hdGUpCmBgYAoKIyMgUHJlZGljdGl2ZSBwb3dlciBhbmQgcGVycGxleGl0eQoKYGBge3J9Cm1vZGVsX2RlbHRhcyAlPiUKICBtdXRhdGUodGVzdF9wcGwgPSBpZl9lbHNlKHRlc3RfcHBsID4gNTAwLCAzMjkuOSwgdGVzdF9wcGwpKSAlPiUKICBnZ3Bsb3QoYWVzKHg9dGVzdF9wcGwsIHk9ZGVsdGFfdGVzdF9tZWFuLCBjb2xvcj10cmFpbmluZ192b2NhYiwgZmlsbCA9IHRyYWluaW5nX3ZvY2FiLCB5bWluPTApKSArCiAgICB0aGVtZV9idygpICsKICAgIGdlb21fdGV4dChhZXMoeD0yNzUsIHk9MCwgbGFiZWwgPSBjKCIvLyIpKSkgKwogICAgZ2VvbV9lcnJvcmJhcihhZXMoeW1pbj1kZWx0YV90ZXN0X21lYW4tZGVsdGFfdGVzdF9zZW0sIHltYXg9ZGVsdGFfdGVzdF9tZWFuK2RlbHRhX3Rlc3Rfc2VtKSwgYWxwaGE9MC40KSArCiAgICAjZ2VvbV9zbW9vdGgobWV0aG9kPSJsbSIsIHNlPUYpICsKICAgIGdlb21fcG9pbnQoc3RhdD0iaWRlbnRpdHkiLCBwb3NpdGlvbj0iZG9kZ2UiLCBhbHBoYT0xLCBzaXplPTQsIGFlcyhzaGFwZT1tb2RlbCwgY29sb3IgPSB0cmFpbmluZ192b2NhYikpICsKICAgIHlsYWIobWV0cmljKSArCiAgICB4bGFiKCJUZXN0IFBlcnBsZXhpdHkiKSArCiAgICAjY29vcmRfY2FydGVzaWFuKHlsaW0gPSBjKDEsIDE2KSkgKwogICAgZ2d0aXRsZSgiVGVzdCBQZXJwbGV4aXR5IHZzLiBQcmVkaWN0aXZlIFBvd2VyIikgKwogICAgc2NhbGVfY29sb3JfbWFudWFsKHZhbHVlcyA9IGMoImJsbGlwLWxnIj0iIzQ0MDE1NEZGIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJibGxpcC1tZCI9IiMzOTU2OENGRiIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiYmxsaXAtc20iPSIjMUY5NjhCRkYiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImJsbGlwLXhzIj0iIzczRDA1NUZGIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJncHRicGUiPSIjZjA5NDFmIikpICsKICAgIHNjYWxlX3NoYXBlX21hbnVhbCh2YWx1ZXMgPSBjKDE2LCAxNywgMTUsIDE4KSkgKwogICAgc2NhbGVfeF9jb250aW51b3VzKGxhYmVscz1jKDAsIDUwLCAxMDAsIDE1MCwgMjAwLCAyNTAsIDUwMCAsNTUwKSwgYnJlYWtzPWMoMCwgNTAsIDEwMCwgMTUwLCAyMDAsIDI1MCwgMzAwLCAzNTApLCBtaW5vcl9icmVha3MgPSBOVUxMKSArCiAgICBzY2FsZV95X2NvbnRpbnVvdXMobGltaXRzID0gYygwLCBOQSksIGV4cGFuZCA9IGMoMCwwKSkgKwogICAgZmFjZXRfd3JhcCh+Y29ycHVzLCBzY2FsZXM9ImZyZWUiKSArCiAgICBjb29yZF9jYXJ0ZXNpYW4oY2xpcD0ib2ZmIikgKwogICAgdGhlbWUoYXhpcy50ZXh0PWVsZW1lbnRfdGV4dChzaXplPTEyKSwKICAgICAgICAgIHN0cmlwLnRleHQueCA9IGVsZW1lbnRfdGV4dChzaXplPTEyKSwKICAgICAgICAgIGxlZ2VuZC50ZXh0PWVsZW1lbnRfdGV4dChzaXplPTEyKSwKICAgICAgICAgIGF4aXMudGl0bGU9ZWxlbWVudF90ZXh0KHNpemU9MTIpLAogICAgICAgICAgbGVnZW5kLnBvc2l0aW9uID0gInJpZ2h0IikKZ2dzYXZlKCIuLi9pbWFnZXMvY3VueTIwMjAvcHBsX2xvZ2xpay5wbmciLGhlaWdodD00LjIsd2lkdGg9MTIpCgpgYGAKCmBgYHtyfQoKZGxsX2Nvcl90ZXN0ID0gZnVuY3Rpb24oZGYpewogIGRmICU+JQogICAgc3VtbWFyaXNlKAogICAgICBjb3IgPSBjb3IudGVzdChkZiRkZWx0YV90ZXN0X21lYW4sIGRmJHRlc3RfcHBsKSRlc3RpbWF0ZSwKICAgICAgcCA9IGNvci50ZXN0KGRmJGRlbHRhX3Rlc3RfbWVhbiwgZGYkdGVzdF9wcGwpJHAudmFsdWUKICAgICkKfQoKbW9kZWxfZGVsdGFzICU+JQogIGZpbHRlcihtb2RlbCAhPSAiNWdyYW0iKSAlPiUKICBncm91cF9ieSh0cmFpbmluZywgY29ycHVzKSAlPiUKICAgIG11dGF0ZShuID0gbigpKSAlPiUKICB1bmdyb3VwKCkgJT4lCiAgZmlsdGVyKG4gPiAyKSAlPiUKICBncm91cF9ieSh0cmFpbmluZywgY29ycHVzKSAlPiUKICAgIGRvKHsgZGxsX2Nvcl90ZXN0KC4pIH0pICU+JQogIHVuZ3JvdXAoKSAlPiUKICBhcnJhbmdlKGNvcnB1cykKCmBgYAoKIyMgRWZmZWN0IG9mIHRyYWluaW5nIGRhdGEgc2l6ZQoKYGBge3IgT24gcHJlZGljdGl2ZSBwb3dlcn0KbW9kZWxfZGVsdGFzICU+JQogIG11dGF0ZSh0cmFpbl9zaXplID0gbG9nKHRyYWluX3NpemUpKSAlPiUKICBtdXRhdGUoYnBlID0gaWZfZWxzZSh0cmFpbmluZ192b2NhYiA9PSAiZ3B0YnBlIiwgInllcyIsICJubyIpLAogICAgICAgICBicGUgPSBhcy5mYWN0b3IoYnBlKSkgJT4lCiAgZ2dwbG90KGFlcyh4PXRyYWluX3NpemUsIHk9ZGVsdGFfdGVzdF9tZWFuLCBjb2xvcj1tb2RlbCkpICsKICAgIHRoZW1lX2J3KCkgKwogICAgZ2VvbV9lcnJvcmJhcihhZXMoeW1pbj1kZWx0YV90ZXN0X21lYW4tZGVsdGFfdGVzdF9zZW0sIHltYXg9ZGVsdGFfdGVzdF9tZWFuK2RlbHRhX3Rlc3Rfc2VtKSwgd2lkdGggPSAwLjEpICsKICAgIGdlb21fc21vb3RoKG1ldGhvZD0ibG0iLCBzZT1ULCBhbHBoYT0wLjIpICsKICAgIGdlb21fcG9pbnQoc3RhdD0iaWRlbnRpdHkiLCBwb3NpdGlvbj0iZG9kZ2UiLCBhbHBoYT0xLCBzaXplPTMsIGFlcyhzaGFwZT1icGUpKSArCiAgICB5bGFiKG1ldHJpYykgKwogICAgeGxhYigiTG9nIE1pbGxpb24gVHJhaW5pbmcgVG9rZW5zIikgKwogICAgZ2d0aXRsZSgiVHJhaW5pbmcgU2l6ZSB2cy4gUHJlZGljdGl2ZSBQb3dlciIpICsKICAgIGZhY2V0X2dyaWQoLn5jb3JwdXMsIHNjYWxlcz0iZnJlZSIpICsKICAgICNzY2FsZV9jb2xvcl9tYW51YWwodmFsdWVzID0gYygiI0E0MkVGMSIsICIjMzg5NEM4IikpICsKICAgIHRoZW1lKGF4aXMudGV4dD1lbGVtZW50X3RleHQoc2l6ZT0xMiksCiAgICAgICAgICBzdHJpcC50ZXh0LnggPSBlbGVtZW50X3RleHQoc2l6ZT0xMiksCiAgICAgICAgICBsZWdlbmQudGV4dD1lbGVtZW50X3RleHQoc2l6ZT04KSwKICAgICAgICAgIGxlZ2VuZC50aXRsZT1lbGVtZW50X3RleHQoc2l6ZT04KSwKICAgICAgICAgIGF4aXMudGl0bGU9ZWxlbWVudF90ZXh0KHNpemU9MTQpLAogICAgICAgICAgbGVnZW5kLnBvc2l0aW9uID0gImJvdHRvbSIsCiAgICAgICAgICBsZWdlbmQuZGlyZWN0aW9uID0gImhvcml6b250YWwiLAogICAgICAgICAgbGVnZW5kLmtleS53aWR0aCA9IHVuaXQoMC4zLCJjbSIpLAogICAgICAgICAgbGVnZW5kLnNwYWNpbmcueCA9IHVuaXQoMC4xLCAnY20nKSkKI2dnc2F2ZSgiLi4vaW1hZ2VzL2N1bnkyMDIwL3RyYWluaW5nX2xvZ2xpay5wbmciLGhlaWdodD01LHdpZHRoPTUpCmBgYAoKYGBge3J9Cgptb2RlbF9jb3JfdGVzdCA9IGZ1bmN0aW9uKGRmKXsKICBkZiAlPiUKICAgIHN1bW1hcmlzZShjb3IgPSBjb3IudGVzdChkZiR0cmFpbl9zaXplLCBkZiRkZWx0YV90ZXN0X21lYW4pJGVzdGltYXRlLAogICAgICAgICAgICAgIHAgPSBjb3IudGVzdChkZiR0cmFpbl9zaXplLCBkZiRkZWx0YV90ZXN0X21lYW4pJHAudmFsdWUpCn0KCm1vZGVsX2RlbHRhcyAlPiUKICBncm91cF9ieShtb2RlbCwgY29ycHVzKSAlPiUKICAgIGRvKHttb2RlbF9jb3JfdGVzdCguKX0pICU+JQogIHVuZ3JvdXAoKSAlPiUKICBhcnJhbmdlKCkKCgpgYGAKCmBgYHtyIFBQTCB2cy4gU0cgc2NvcmV9Cm1vZGVsX2RlbHRhcyAlPiUKICBtdXRhdGUodGVzdF9wcGwgPSBpZl9lbHNlKHRlc3RfcHBsID4gNTAwLCAzMjkuOSwgdGVzdF9wcGwpKSAlPiUKICBtdXRhdGUodHJhaW5fc2l6ZSA9IGxvZyh0cmFpbl9zaXplKSkgJT4lCiAgbXV0YXRlKGJwZSA9IGlmX2Vsc2UodHJhaW5pbmdfdm9jYWIgPT0gImdwdGJwZSIsICJ5ZXMiLCAibm8iKSwKICAgICAgICAgYnBlID0gYXMuZmFjdG9yKGJwZSkpICU+JQogIGdncGxvdChhZXMoeD10ZXN0X3BwbCwgeT1zZ19zY29yZSwgY29sb3I9dHJhaW5pbmdfdm9jYWIpKSArCiAgICB0aGVtZV9idygpICsKICAgICNnZW9tX3Ntb290aChtZXRob2Q9ImxtIiwgc2U9VCwgYWxwaGE9MC4yKSArCiAgICBnZW9tX3BvaW50KHN0YXQ9ImlkZW50aXR5IiwgcG9zaXRpb249ImRvZGdlIiwgYWxwaGE9MC42LCBzaXplPTUsIGFlcyhzaGFwZT1tb2RlbCkpICsKICAgIGdlb21fdGV4dChhZXMoeD0yNzUsIHk9MCwgbGFiZWwgPSBjKCIvLyIpKSkgKwogICAgeWxhYigiU0cgU0NvcmUiKSArCiAgICB4bGFiKCJUZXN0IFBlcnBsZXhpdHkiKSArCiAgICBnZ3RpdGxlKCJUZXN0IFBQTCB2cy4gU0cgU2NvcmUiKSArCiAgICBzY2FsZV9jb2xvcl9tYW51YWwodmFsdWVzID0gYygiYmxsaXAtbGciPSIjNDQwMTU0RkYiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImJsbGlwLW1kIj0iIzM5NTY4Q0ZGIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJibGxpcC1zbSI9IiMxRjk2OEJGRiIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiYmxsaXAteHMiPSIjNzNEMDU1RkYiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImdwdGJwZSI9IiNmMDk0MWYiKSkgKwogICAgc2NhbGVfc2hhcGVfbWFudWFsKHZhbHVlcyA9IGMoMTYsIDE3LCAxNSwgMTgpKSArCiAgICBzY2FsZV94X2NvbnRpbnVvdXMobGFiZWxzPWMoMCwgNTAsIDEwMCwgMTUwLCAyMDAsIDI1MCwgNTAwICw1NTApLCBicmVha3M9YygwLCA1MCwgMTAwLCAxNTAsIDIwMCwgMjUwLCAzMDAsIDM1MCksIG1pbm9yX2JyZWFrcyA9IE5VTEwpICsKICAgIHNjYWxlX3lfY29udGludW91cyhsaW1pdHMgPSBjKDAsIDEpLCBleHBhbmQgPSBjKDAsMCkpICsKICAgIHRoZW1lKGF4aXMudGV4dD1lbGVtZW50X3RleHQoc2l6ZT0xMiksCiAgICAgICAgICBzdHJpcC50ZXh0LnggPSBlbGVtZW50X3RleHQoc2l6ZT0xMiksCiAgICAgICAgICBsZWdlbmQudGV4dD1lbGVtZW50X3RleHQoc2l6ZT04KSwKICAgICAgICAgIGxlZ2VuZC50aXRsZT1lbGVtZW50X3RleHQoc2l6ZT04KSwKICAgICAgICAgIGF4aXMudGl0bGU9ZWxlbWVudF90ZXh0KHNpemU9MTQpLAogICAgICAgICAgbGVnZW5kLnBvc2l0aW9uID0gIm5vbmUiLAogICAgICAgICAgbGVnZW5kLmRpcmVjdGlvbiA9ICJob3Jpem9udGFsIiwKICAgICAgICAgIGxlZ2VuZC5rZXkud2lkdGggPSB1bml0KDAuMywiY20iKSwKICAgICAgICAgIGxlZ2VuZC5zcGFjaW5nLnggPSB1bml0KDAuMSwgJ2NtJykpCmdnc2F2ZSgiLi4vaW1hZ2VzL2N1bnkyMDIwL3BwbF9zZy5wbmciLGhlaWdodD00LjUsd2lkdGg9MykKYGBgCgoKYGBge3IgU2l6ZSB2cy4gU0cgc2NvcmV9Cm1vZGVsX2RlbHRhcyAlPiUKICBtdXRhdGUodHJhaW5fc2l6ZSA9IGxvZyh0cmFpbl9zaXplKSkgJT4lCiAgbXV0YXRlKGJwZSA9IGlmX2Vsc2UodHJhaW5pbmdfdm9jYWIgPT0gImdwdGJwZSIsICJ5ZXMiLCAibm8iKSwKICAgICAgICAgYnBlID0gYXMuZmFjdG9yKGJwZSkpICU+JQogIGdncGxvdChhZXMoeD10cmFpbl9zaXplLCB5PXNnX3Njb3JlLCBjb2xvcj1tb2RlbCkpICsKICAgIHRoZW1lX2J3KCkgKwogICAgZ2VvbV9zbW9vdGgobWV0aG9kPSJsbSIsIHNlPVQsIGFscGhhPTAuMikgKwogICAgZ2VvbV9wb2ludChzdGF0PSJpZGVudGl0eSIsIHBvc2l0aW9uPSJkb2RnZSIsIGFscGhhPTEsIHNpemU9MywgYWVzKHNoYXBlPWJwZSkpICsKICAgIHlsYWIoIlNHIFNDb3JlIikgKwogICAgeGxhYigiTG9nIE1pbGxpb24gVHJhaW5pbmcgVG9rZW5zIikgKwogICAgZ2d0aXRsZSgiVHJhaW5pbmcgU2l6ZSB2cy4gU0cgU2NvcmUiKSArCiAgICAjc2NhbGVfY29sb3JfbWFudWFsKHZhbHVlcyA9IGMoIiNBNDJFRjEiLCAiIzM4OTRDOCIpKSArCiAgICAjZmFjZXRfZ3JpZCh+bW9kZWwsIHNjYWxlcz0iZnJlZSIpICsKICAgIHRoZW1lKGF4aXMudGV4dD1lbGVtZW50X3RleHQoc2l6ZT0xMiksCiAgICAgICAgICBzdHJpcC50ZXh0LnggPSBlbGVtZW50X3RleHQoc2l6ZT0xMiksCiAgICAgICAgICBsZWdlbmQudGV4dD1lbGVtZW50X3RleHQoc2l6ZT04KSwKICAgICAgICAgIGxlZ2VuZC50aXRsZT1lbGVtZW50X3RleHQoc2l6ZT04KSwKICAgICAgICAgIGF4aXMudGl0bGU9ZWxlbWVudF90ZXh0KHNpemU9MTQpLAogICAgICAgICAgbGVnZW5kLnBvc2l0aW9uID0gImJvdHRvbSIsCiAgICAgICAgICBsZWdlbmQuZGlyZWN0aW9uID0gImhvcml6b250YWwiLAogICAgICAgICAgbGVnZW5kLmtleS53aWR0aCA9IHVuaXQoMC4zLCJjbSIpLAogICAgICAgICAgbGVnZW5kLnNwYWNpbmcueCA9IHVuaXQoMC4xLCAnY20nKSkKI2dnc2F2ZSgiLi4vaW1hZ2VzL2N1bnkyMDIwL3RyYWluaW5nX3NnLnBuZyIsaGVpZ2h0PTUsd2lkdGg9NCkKYGBgCgoKYGBge3J9Cgptb2RlbF9jb3JfdGVzdCA9IGZ1bmN0aW9uKGRmKXsKICBkZiAlPiUKICAgIHN1bW1hcmlzZShjb3IgPSBjb3IudGVzdChkZiR0cmFpbl9zaXplLCBkZiRzZ19zY29yZSkkZXN0aW1hdGUsCiAgICAgICAgICAgICAgcCA9IGNvci50ZXN0KGRmJHRyYWluX3NpemUsIGRmJHNnX3Njb3JlKSRwLnZhbHVlKQp9Cgptb2RlbF9kZWx0YXMgJT4lCiAgZ3JvdXBfYnkobW9kZWwpICU+JQogICAgZG8oe21vZGVsX2Nvcl90ZXN0KC4pfSkgJT4lCiAgdW5ncm91cCgpCgoKCmBgYAoKIyMgU21pdGggJiBMZXZ5IHJlcHJvZHVjdGlvbgoKIyMjIyBUaGlzIHJlZG9uZSBzbyB0aGF0IGl0J3MgdW5pcXVlIGZvciBlYWNoIG1vZGVsCmBgYHtyfQphbGxfZGF0YSAlPiUKICBnZ3Bsb3QoYWVzKHg9c3VycHJpc2FsLCBjb2xvcj1tb2RlbCkpICsKICB0aGVtZV9idygpICsKICBnZW9tX2RlbnNpdHkoKSArCiAgZmFjZXRfZ3JpZCh+Y29ycHVzKSArCiAgY29vcmRfY2FydGVzaWFuKHhsaW0gPSBjKDAsIDIxKSkgKwogIHRoZW1lKHBhbmVsLnNwYWNpbmcgPSB1bml0KDIuNSwgImNtIikpCmdnc2F2ZSgiLi4vaW1hZ2VzL2N1bnkyMDIwL3N1cnBfY29ycl9tYXJnaW5hbHMucG5nIixoZWlnaHQ9MS41LHdpZHRoPTExKQoKYGBgCgoKYGBge3IgRml0IEdBTXN9CmsgPSAxLjk3CgojIEZpdCBhIEdBTSBmb3IgYSBib290c3RyYXAgc2FtcGxlLgpmaXRfZ2FtX2lubmVyID0gZnVuY3Rpb24oYm9vdHN0cmFwX3NhbXBsZSwga2V5KSB7CiAgZGYgPSBhbmFseXNpcyhib290c3RyYXBfc2FtcGxlKQogIAogIGlmIChrZXkkY29ycHVzID09ICJkdW5kZWUiKSB7CiAgICAjIFJlYWRpbmcgdGltZSByZWdyZXNzaW9uOiB1c2UgZmVhdHVyZXMgb2YgY3VycmVudCBhbmQgcHJldmlvdXMgd29yZAogICAgbSA9IGdhbShwc3ljaG9tZXRyaWMgfiBzKHN1cnByaXNhbCwgYnMgPSAnY3InLCBrID0gMjApICsgcyhwcmV2X3N1cnAsIGJzID0gJ2NyJywgayA9IDIwKSArIHRlKGZyZXEsIGxlbiwgYnMgPSAnY3InKSArIHRlKHByZXZfZnJlcSwgcHJldl9sZW4sIGJzID0gJ2NyJyksIGRhdGEgPSBkZikKICB9IGVsc2UgewogICAgIyBTUFJUIHJlZ3Jlc3Npb246IHVzZSBmZWF0dXJlcyBvZiBjdXJyZW50IGFuZCAzIHByZXZpb3VzIHdvcmRzCiAgICBtID0gZ2FtKHBzeWNob21ldHJpYyB+IHMoc3VycHJpc2FsLCBicyA9ICdjcicsIGsgPSAyMCkgKyBzKHByZXZfc3VycCwgYnMgPSAnY3InLCBrID0gMjApICsgcyhwcmV2Ml9zdXJwLCBicyA9ICdjcicsIGsgPSAyMCkgKyBzKHByZXYzX3N1cnAsIGJzID0gJ2NyJywgayA9IDIwKSArIHRlKGZyZXEsIGxlbiwgYnMgPSAnY3InKSArIHRlKHByZXZfZnJlcSwgcHJldl9sZW4sIGJzID0gJ2NyJykgKyB0ZShwcmV2Ml9mcmVxLCBwcmV2Ml9sZW4sIGJzID0gJ2NyJykgKyB0ZShwcmV2M19mcmVxLCBwcmV2M19sZW4sIGJzID0gJ2NyJyksIGRhdGEgPSBkZikKICB9CgogICMgUHJvZHVjZSBhIG1vZGVsIGZpdCBsaW5lCiAgbmV3ZGF0YSA9IGRhdGEuZnJhbWUoc3VycHJpc2FsPXNlcSgwLDIwLGJ5PTAuMSksCiAgICAgICAgICAgICAgICAgICAgICAgcHJldl9zdXJwPXNlcSgwLDIwLGJ5PTAuMSksCiAgICAgICAgICAgICAgICAgICAgICAgcHJldjJfc3VycD1zZXEoMCwyMCxieT0wLjEpLAogICAgICAgICAgICAgICAgICAgICAgIHByZXYzX3N1cnA9c2VxKDAsMjAsYnk9MC4xKSwKICAgICAgICAgICAgICAgICAgICAgICBmcmVxPTAsIHByZXZfZnJlcT0wLCBwcmV2Ml9mcmVxPTAsIHByZXYzX2ZyZXE9MCwKICAgICAgICAgICAgICAgICAgICAgICBsZW49MCwgcHJldl9sZW49MCwgcHJldjJfbGVuPTAsIHByZXYzX2xlbj0wKQogCiAgbmV3ZGF0YSA9IG5ld2RhdGEgJT4lCiAgICBtdXRhdGUoeT1wcmVkaWN0KG0sIG5ld2RhdGE9LiwgdHlwZT0ibGluayIsCiAgICAgICAgICAgICAgICAgICAgIGV4Y2x1ZGU9YygidGUoZnJlcSwgbGVuLCBicyA9ICdjcicpIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJ0ZShwcmV2X2ZyZXEsIHByZXZfbGVuLCBicyA9ICdjcicpIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJ0ZShwcmV2Ml9mcmVxLCBwcmV2Ml9sZW4sIGJzID0gJ2NyJykiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgInRlKHByZXYzX2ZyZXEsIHByZXYzX2xlbiwgYnMgPSAnY3InKSIpKSkKICAKICByZXR1cm4obmV3ZGF0YSkKfQoKIyBGaXQgYSBib290c3RyYXAtcmUtZXN0aW1hdGVkIEdBTSBmb3IgdGhlIGdpdmVuIG1vZGVsLS1jb3JwdXMtLXRyYWluaW5nIGdyb3VwLgpmaXRfZ2FtID0gZnVuY3Rpb24oZGYsIGtleSwgYWxwaGE9MC4wNSkgewogICMgQm9vdHN0cmFwLXJlc2FtcGxlIGRhdGEKICBib290X21vZGVscyA9IGRmICU+JSBib290c3RyYXBzKHRpbWVzPTEwMCkgJT4lIAogICAgIyBGaXQgYSBHQU0gYW5kIGdldCBwcmVkaWN0aW9ucyBmb3IgZWFjaCBzYW1wbGUKICAgIG11dGF0ZShzbW9vdGhlZD1tYXAoc3BsaXRzLCBmaXRfZ2FtX2lubmVyLCBrZXk9a2V5KSkKICAKICAjIEV4dHJhY3QgbWVhbiBhbmQgNSUgYW5kIDk1JSBwZXJjZW50aWxlIHktdmFsdWVzIGZvciBlYWNoIHN1cnByaXNhbCB2YWx1ZQogIHJlc3VsdCA9IGJvb3RfbW9kZWxzICU+JSAKICAgIHVubmVzdChzbW9vdGhlZCkgJT4lIAogICAgc2VsZWN0KHN1cnByaXNhbCwgeSkgJT4lIAogICAgZ3JvdXBfYnkoc3VycHJpc2FsKSAlPiUgCiAgICAgIHN1bW1hcmlzZSh5X2xvd2VyPXF1YW50aWxlKHksIGFscGhhIC8gMiksIAogICAgICAgICAgICAgICAgeV91cHBlcj1xdWFudGlsZSh5LCAxIC0gYWxwaGEgLyAyKSwKICAgICAgICAgICAgICAgIHk9bWVhbih5KSkgJT4lIAogICAgdW5ncm91cCgpCiAgCiAgcmV0dXJuIChyZXN1bHQpCn0KCnNtb290aHMgPSBhbGxfZGF0YSAlPiUKICBtdXRhdGUoCiAgICB0cmFpbmluZ192b2NhYj1hcy5mYWN0b3IoaWZlbHNlKHN0cl9kZXRlY3QodHJhaW5pbmcsICJncHRicGUiKSwgImdwdGJwZSIsIGFzLmNoYXJhY3Rlcih0cmFpbmluZykpKSwKICAgIHRyYWluaW5nX3NvdXJjZT1hcy5mYWN0b3Ioc3RyX3JlcGxhY2UoYXMuY2hhcmFjdGVyKHRyYWluaW5nKSwgIi1ncHRicGUiLCAiIikpKSAlPiUKICBncm91cF9ieSh0cmFpbmluZ192b2NhYiwgbW9kZWwsIGNvcnB1cykgJT4lCiAgICBncm91cF9tb2RpZnkoZml0X2dhbSkgJT4lCiAgdW5ncm91cCgpCndyaXRlLmNzdihzbW9vdGhzLCAiLi4vZGF0YS9nYW1fc21vb3Rocy5jc3YiKQoKCmBgYAoKCmBgYHtyfQoKc21vb3RocyAlPiUKICAjbXV0YXRlKHRyYWluaW5nX21vZGVsID0gcGFzdGUodHJhaW5pbmcsICJfIiwgbW9kZWwsIHNlcD0iIikpICU+JQogICNmaWx0ZXIodHJhaW5pbmdfdm9jYWIgPT0gImJsbGlwLXhzIiB8IHRyYWluaW5nX3ZvY2FiID09ICJncHRicGUiIHwgdHJhaW5pbmdfdm9jYWIgPT0gImJsbGlwLWxnIikgJT4lCiAgI2ZpbHRlcih0cmFpbmluZyAhPSAiYmxsaXAtbWQtZ3B0YnBlIiAmIHRyYWluaW5nICE9ICJibGxpcC1zbS1ncHRicGUiKSAlPiUKICAjIG11dGF0ZSh0cmFpbmluZyA9IGFzLmNoYXJhY3Rlcih0cmFpbmluZyksCiAgIyAgICAgICAgdHJhaW5pbmcgPSBpZl9lbHNlKHRyYWluaW5nID09ICJibGxpcC1sZy1ncHRicGUiLCAiYnBlIFxuIGJsbGlwLWxnIiwgdHJhaW5pbmcpLAogICMgICAgICAgIHRyYWluaW5nID0gaWZfZWxzZSh0cmFpbmluZyA9PSAiYmxsaXAtbWQtZ3B0YnBlIiwgImJwZSBcbiBibGxpcC1tZCIsIHRyYWluaW5nKSwKICAjICAgICAgICB0cmFpbmluZyA9IGlmX2Vsc2UodHJhaW5pbmcgPT0gImJsbGlwLXNtLWdwdGJwZSIsICJicGUgXG4gYmxsaXAtc20iLCB0cmFpbmluZyksCiAgIyAgICAgICAgdHJhaW5pbmcgPSBpZl9lbHNlKHRyYWluaW5nID09ICJibGxpcC14cy1ncHRicGUiLCAiYnBlIFxuIGJsbGlwLXhzIiwgdHJhaW5pbmcpKSAlPiUKICBnZ3Bsb3QoYWVzKHg9c3VycHJpc2FsLCB5PXksIGZpbGw9dHJhaW5pbmdfdm9jYWIsIGxpbmV0eXBlPW1vZGVsKSkgKwogICAgICB0aGVtZV9idygpICsKICAgICAgZ2VvbV9saW5lKHNpemU9MC41LCBhZXMoY29sb3I9dHJhaW5pbmdfdm9jYWIpKSArCiAgICAgICNnZW9tX2xpbmUoYWVzKHk9eV9sb3dlciksIGxpbmV0eXBlPSJkYXNoZWQiKSArCiAgICAgIGdlb21fcmliYm9uKGFlcyh5bWluPXlfbG93ZXIseW1heD15X3VwcGVyKSwgYWxwaGE9MC4zKSArCiAgICAgICNnZW9tX2xpbmUoYWVzKHk9eV91cHBlciksIGxpbmV0eXBlPSJkYXNoZWQiKSArCiAgICAgIGZhY2V0X2dyaWQoY29ycHVzIH4gdHJhaW5pbmdfdm9jYWIgKyBtb2RlbCwgc2NhbGVzPSJmcmVlIikgKwogICAgICBzY2FsZV9jb2xvcl9tYW51YWwodmFsdWVzID0gYygiYmxsaXAtbGciPSIjNDQwMTU0RkYiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImJsbGlwLW1kIj0iIzM5NTY4Q0ZGIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJibGxpcC1zbSI9IiMxRjk2OEJGRiIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiYmxsaXAteHMiPSIjNzNEMDU1RkYiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImdwdGJwZSI9IiNmMDk0MWYiKSkgKwogICAgICBzY2FsZV9maWxsX21hbnVhbCh2YWx1ZXMgPSBjKCJibGxpcC1sZyI9IiM0NDAxNTRGRiIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiYmxsaXAtbWQiPSIjMzk1NjhDRkYiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImJsbGlwLXNtIj0iIzFGOTY4QkZGIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJibGxpcC14cyI9IiM3M0QwNTVGRiIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiZ3B0YnBlIj0iI2YwOTQxZiIpKSArCiAgICAgIHNjYWxlX3hfY29udGludW91cyhsYWJlbHM9YygwLCAxMCwgMjApLCBicmVha3M9YygwLCAxMCwgMjApLCBtaW5vcl9icmVha3MgPSBOVUxMKSArCiAgICAgIHlsYWIoIlJlYWRpbmcgVGltZSIpICsKICAgICAgdGhlbWUobGVnZW5kLnBvc2l0aW9uID0gImJvdHRvbSIpCmdnc2F2ZSgiLi4vaW1hZ2VzL2N1bnkyMDIwL2dhbV9zdXJwX2NvcnIucG5nIiwgaGVpZ2h0PTUsd2lkdGg9MTIpCgoKCmBgYAoKCiMjIyBPbGQgUGxvdHRpbmcKCmBgYHtyfQphbGxfZGF0YSAlPiUKICBmaWx0ZXIobW9kZWwgPT0gImdwdDIiLCBjb3JwdXMgPT0gImR1bmRlZSIpICU+JQogIGZpbHRlcihzdXJwcmlzYWw8MjEpICU+JQogIG11dGF0ZShicGU9c3RyX2RldGVjdCh0cmFpbmluZywgImJwZSIpLAogICAgICAgICB0cmFpbmluZ19zb3VyY2U9c3RyX3JlcGxhY2UodHJhaW5pbmcsICItZ3B0YnBlIiwgIiIpKSAlPiUgCiAgZ2dwbG90KGFlcyh4PXN1cnByaXNhbCwgeT1wc3ljaG9tZXRyaWMsIGNvbG9yPXRyYWluaW5nX3NvdXJjZSwgbGluZXR5cGU9YnBlKSkgKwogICAgdGhlbWVfYncoKSArCiAgICAjc3RhdF9zbW9vdGgoc2U9VCwgYWxwaGE9MC41KSArCiAgICBnZW9tX3Ntb290aChtZXRob2QgPSAiZ2FtIiwgZm9ybXVsYSA9IHBzeWNob21ldHJpYyB+IHMoc3VycHJpc2FsLCBicyA9ICdjcicsIGsgPSAyMCkgKyBzKHByZXZfc3VycCwgYnMgPSAnY3InLCBrID0gMjApICsgdGUoZnJlcSwgbGVuLCBicyA9ICdjcicpICsgdGUocHJldl9mcmVxLCBwcmV2X2xlbiwgYnMgPSAnY3InLCBzZSA9IEYpKSArCiAgICAjZ2VvbV9lcnJvcmJhcihjb2xvcj0iYmxhY2siLCB3aWR0aD0uMiwgcG9zaXRpb249cG9zaXRpb25fZG9kZ2Uod2lkdGg9LjkpLCBhbHBoYT0wLjMpICsKICAgICNnZW9tX3BvaW50KHN0YXQ9ImlkZW50aXR5IiwgcG9zaXRpb249ImRvZGdlIiwgYWxwaGE9MSwgc2l6ZT0zKSArCiAgICB5bGFiKCJQcm9jZXNzaW5nIFRpbWUgKG1zKSIpICsKICAgIHhsYWIoIlN1cnByaXNhbCAoYml0cykiKSArCiAgICBnZ3RpdGxlKCJTdXJwcmlzYWwgdnMuIFJlYWRpbmcgVGltZSAvIEdhemUgRHVyYXRpb24iKSArCiAgICBmYWNldF93cmFwKG1vZGVsIH4gY29ycHVzLCBzY2FsZXM9ImZyZWUiLCBuY29sPTMsIHN0cmlwLnBvc2l0aW9uID0gYygicmlnaHQiKSkgKwogICAgc2NhbGVfY29sb3JfbWFudWFsKHZhbHVlcyA9IGMoImJsbGlwLWxnIj0iIzQ0MDE1NEZGIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJibGxpcC1tZCI9IiMzOTU2OENGRiIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiYmxsaXAtc20iPSIjMUY5NjhCRkYiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImJsbGlwLXhzIj0iIzczRDA1NUZGIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJibGxpcC1sZy1ncHRicGUiPSIjODg4ODg4IiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJibGxpcC1tZC1ncHRicGUiPSIjODg4ODg4IiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJibGxpcC1zbS1ncHRicGUiPSIjODg4ODg4IiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJibGxpcC14cy1ncHRicGUiPSIjODg4ODg4IikpICsKICAgIGNvb3JkX2NhcnRlc2lhbih4bGltID0gYygwLCAyMSkpICsKICAgIHRoZW1lKGF4aXMudGV4dD1lbGVtZW50X3RleHQoc2l6ZT0xMCksCiAgICAgICAgICBheGlzLnRleHQueSA9IGVsZW1lbnRfdGV4dChzaXplID0gMTApLAogICAgICAgICAgc3RyaXAudGV4dC54ID0gZWxlbWVudF90ZXh0KHNpemU9MTApLAogICAgICAgICAgbGVnZW5kLnRleHQ9ZWxlbWVudF90ZXh0KHNpemU9MTApLAogICAgICAgICAgYXhpcy50aXRsZT1lbGVtZW50X3RleHQoc2l6ZT0xMiksCiAgICAgICAgICBsZWdlbmQucG9zaXRpb24gPSAicmlnaHQiKQojZ2dzYXZlKCIuLi9pbWFnZXMvY3VueTIwMjAvc3VycF9jb3JyLnBuZyIsaGVpZ2h0PTYsd2lkdGg9MTIpCmBgYAoKCmBgYHtyfQoKY29ycl90ZXN0ID0gZnVuY3Rpb24oZGYpewogIGRmICU+JQogICAgc3VtbWFyaXNlKAogICAgICBjb3IgPSBjb3IudGVzdChkZiRzdXJwcmlzYWwsIGRmJHBzeWNob21ldHJpYykkZXN0aW1hdGUKICAgICkKfQoKYWxsX2RhdGEgJT4lCiAgZ3JvdXBfYnkobW9kZWwsIHRyYWluaW5nLCBjb3JwdXMsIHNlZWQpICU+JQogICAgZG8oeyBjb3IgPSBjb3JyX3Rlc3QoLil9KSAlPiUKICB1bmdyb3VwKCkKCmBgYAoKCgojIyMgSW52ZXN0aWdhdGUgdmFuaWxsYQoKYGBge3J9CmFsbF9kYXRhICU+JQogICNmaWx0ZXIoc3VycHJpc2FsIDwgMTUsIHN1cnByaXNhbCA+IDApICU+JQogIGZpbHRlcihtb2RlbCA9PSAidmFuaWxsYSIpICU+JSAKICBnZ3Bsb3QoYWVzKHg9c3VycHJpc2FsLCB5PXBzeWNob21ldHJpYykpICsKICAgICNzdGF0X3Ntb290aChzZT1ULCBhbHBoYT0wLjUpICsKICAgICNnZW9tX2Vycm9yYmFyKGNvbG9yPSJibGFjayIsIHdpZHRoPS4yLCBwb3NpdGlvbj1wb3NpdGlvbl9kb2RnZSh3aWR0aD0uOSksIGFscGhhPTAuMykgKwogICAgZ2VvbV9wb2ludChhbHBoYT0wLjEpICsgI3N0YXQ9ImlkZW50aXR5IiwgcG9zaXRpb249ImRvZGdlIiwgYWxwaGE9MSwgc2l6ZT0zKSArCiAgICB5bGFiKCJQcm9jZXNzaW5nIFRpbWUgKG1zKSIpICsKICAgIHhsYWIoIlN1cnByaXNhbCAoYml0cykiKSArCiAgICBnZ3RpdGxlKCJTdXJwcmlzYWwgdnMuIFJlYWRpbmcgVGltZSAvIEdhemUgRHVyYXRpb246IFZhbmlsbGEiKSArCiAgICBmYWNldF9ncmlkKGNvcnB1c350cmFpbmluZywgc2NhbGVzID0gImZyZWUiKQogICAgIyBzY2FsZV9jb2xvcl9tYW51YWwodmFsdWVzID0gYygiYmxsaXAtbGciPSIjNDQwMTU0RkYiLAogICAgIyAgICAgICAgICAgICAgICAgICAgICAgICAgICJibGxpcC1tZCI9IiMzOTU2OENGRiIsCiAgICAjICAgICAgICAgICAgICAgICAgICAgICAgICAgImJsbGlwLXNtIj0iIzFGOTY4QkZGIiwKICAgICMgICAgICAgICAgICAgICAgICAgICAgICAgICAiYmxsaXAteHMiPSIjNzNEMDU1RkYiLAogICAgIyAgICAgICAgICAgICAgICAgICAgICAgICAgICJibGxpcC1sZy1ncHRicGUiPSIjODg4ODg4IiwKICAgICMgICAgICAgICAgICAgICAgICAgICAgICAgICAiYmxsaXAtbWQtZ3B0YnBlIj0iIzg4ODg4OCIsCiAgICAjICAgICAgICAgICAgICAgICAgICAgICAgICAgImJsbGlwLXNtLWdwdGJwZSI9IiM4ODg4ODgiLAogICAgIyAgICAgICAgICAgICAgICAgICAgICAgICAgICJibGxpcC14cy1ncHRicGUiPSIjODg4ODg4IikpCmBgYAoKYGBge3IgVG9ueSBCbGFpciBpcyB0aGUgc291cmNlIG9mIHRoYXQgcmlnaHQtc2lkZSBkaXB9CmFsbF9kYXRhICU+JSAKICBmaWx0ZXIoY29ycHVzID09ICJkdW5kZWUiLCBtb2RlbCA9PSAidmFuaWxsYSIsIHRyYWluaW5nID09ICJibGxpcC1sZyIsIHN1cnByaXNhbCA+IDIwLCBwc3ljaG9tZXRyaWMgPCAzMDApCmBgYAoKYGBge3J9CnByaW50KGZ1bGxfcmVzaWR1YWxzICU+JSBmaWx0ZXIoY29ycHVzID09ICJkdW5kZWUiLCBtb2RlbCA9PSAidmFuaWxsYSIsIHRyYWluaW5nID09ICJibGxpcC1sZyIpICU+JSBhcnJhbmdlKGRlc2MocmVzaWQpKSkKZnVsbF9yZXNpZHVhbHMgJT4lIGZpbHRlcihjb3JwdXMgPT0gImR1bmRlZSIsIG1vZGVsID09ICJ2YW5pbGxhIiwgdHJhaW5pbmcgPT0gImJsbGlwLWxnIikgJT4lIGFycmFuZ2UoZGVzYyhyZXNpZCkpICU+JSBmaWx0ZXIocmVzaWQgPiAxNTApICU+JSAKICBnZ3Bsb3QoYWVzKHg9c3VycHJpc2FsKSkgKyBnZW9tX2RlbnNpdHkoKQpgYGAKCgojIyMgSW52ZXN0aWdhdGUgUk5ORwoKYGBge3J9CmFsbF9kYXRhICU+JQogICNmaWx0ZXIoc3VycHJpc2FsIDwgMTUsIHN1cnByaXNhbCA+IDApICU+JQogIGZpbHRlcihtb2RlbCA9PSAicm5uZyIpICU+JSAKICBnZ3Bsb3QoYWVzKHg9c3VycHJpc2FsLCB5PXBzeWNob21ldHJpYykpICsKICAgICNzdGF0X3Ntb290aChzZT1ULCBhbHBoYT0wLjUpICsKICAgICNnZW9tX2Vycm9yYmFyKGNvbG9yPSJibGFjayIsIHdpZHRoPS4yLCBwb3NpdGlvbj1wb3NpdGlvbl9kb2RnZSh3aWR0aD0uOSksIGFscGhhPTAuMykgKwogICAgZ2VvbV9wb2ludChhbHBoYT0wLjEpICsgI3N0YXQ9ImlkZW50aXR5IiwgcG9zaXRpb249ImRvZGdlIiwgYWxwaGE9MSwgc2l6ZT0zKSArCiAgICB5bGFiKCJQcm9jZXNzaW5nIFRpbWUgKG1zKSIpICsKICAgIHhsYWIoIlN1cnByaXNhbCAoYml0cykiKSArCiAgICBnZ3RpdGxlKCJTdXJwcmlzYWwgdnMuIFJlYWRpbmcgVGltZSAvIEdhemUgRHVyYXRpb246IFJOTkciKSArCiAgICBmYWNldF9ncmlkKGNvcnB1c350cmFpbmluZywgc2NhbGVzID0gImZyZWUiKQpgYGAKCmBgYHtyIFRvbnkgQmxhaXIgaXMgdGhlIHNvdXJjZSBvZiB0aGF0IHJpZ2h0LXNpZGUgZGlwIGZvciBSTk5HIHRvb30KYWxsX2RhdGEgJT4lIAogIGZpbHRlcihjb3JwdXMgPT0gImR1bmRlZSIsIG1vZGVsID09ICJybm5nIiwgdHJhaW5pbmcgPT0gImJsbGlwLWxnIiwgc3VycHJpc2FsID4gMjAsIHBzeWNob21ldHJpYyA8IDMwMCkKYGBgCgpgYGB7cn0KcHJpbnQoZnVsbF9yZXNpZHVhbHMgJT4lIGZpbHRlcihjb3JwdXMgPT0gImR1bmRlZSIsIG1vZGVsID09ICJybm5nIiwgdHJhaW5pbmcgPT0gImJsbGlwLWxnIikgJT4lIGFycmFuZ2UoZGVzYyhyZXNpZCkpKQpmdWxsX3Jlc2lkdWFscyAlPiUgZmlsdGVyKGNvcnB1cyA9PSAiZHVuZGVlIiwgbW9kZWwgPT0gInJubmciLCB0cmFpbmluZyA9PSAiYmxsaXAtbGciKSAlPiUgYXJyYW5nZShkZXNjKHJlc2lkKSkgJT4lIGZpbHRlcihyZXNpZCA+IDE1MCkgJT4lIAogIGdncGxvdChhZXMoeD1zdXJwcmlzYWwpKSArIGdlb21fZGVuc2l0eSgpCmBgYAoKIyMjIEludmVzdGlnYXRlIG5ncmFtIHZzIHZhbmlsbGEKCmBgYHtyfQpuZ3JhbV9yZXNpZHMgPSBmdWxsX3Jlc2lkdWFscyAlPiUgZmlsdGVyKG1vZGVsID09ICI1Z3JhbSIsIHRyYWluaW5nID09ICJibGxpcC1zbSIpICU+JSBncm91cF9ieShjb3JwdXMsIGNvZGUpICU+JSBzdW1tYXJpc2UoZnJlcT1tZWFuKGZyZXEpLCBwc3ljaG9tZXRyaWM9bWVhbihwc3ljaG9tZXRyaWMpLCBzdXJwcmlzYWw9bWVhbihzdXJwcmlzYWwpLCByZXNpZD1tZWFuKHJlc2lkKSkKdmFuaWxsYV9yZXNpZHMgPSBmdWxsX3Jlc2lkdWFscyAlPiUgZmlsdGVyKG1vZGVsID09ICJ2YW5pbGxhIiwgdHJhaW5pbmcgPT0gImJsbGlwLXNtIikgJT4lIGdyb3VwX2J5KGNvcnB1cywgY29kZSkgJT4lIHN1bW1hcmlzZShmcmVxPW1lYW4oZnJlcSksIHBzeWNob21ldHJpYz1tZWFuKHBzeWNob21ldHJpYyksIHN1cnByaXNhbD1tZWFuKHN1cnByaXNhbCksICByZXNpZD1tZWFuKHJlc2lkKSkKcmVzaWRzX2pvaW5lZCA9IG5ncmFtX3Jlc2lkcyAlPiUgbGVmdF9qb2luKHZhbmlsbGFfcmVzaWRzLCBieT1jKCJjb3JwdXMiLCAiY29kZSIpLCBzdWZmaXg9YygiLm5ncmFtIiwgIi52YW5pbGxhIikpCgpyZXNpZHNfam9pbmVkICU+JSAKICBnZ3Bsb3QoYWVzKHg9cmVzaWQubmdyYW0sIHk9cmVzaWQudmFuaWxsYSkpICsgZ2VvbV9wb2ludCgpICsgZ2VvbV9hYmxpbmUoc2xvcGU9MSwgY29sb3I9InJlZCIpICsKICBmYWNldF9ncmlkKH5jb3JwdXMpCgpyZXNpZHNfam9pbmVkICU+JSAKICBtdXRhdGUocmVzaWRfZGlmZj1yZXNpZC5uZ3JhbSAtIHJlc2lkLnZhbmlsbGEpICU+JSAKICBnZ3Bsb3QoYWVzKHg9cmVzaWRfZGlmZikpICsgZ2VvbV9kZW5zaXR5KCkgKwogIGZhY2V0X2dyaWQofmNvcnB1cykKCnJlc2lkc19qb2luZWQgJT4lIAogIG11dGF0ZShyZXNpZF9kaWZmPWFicyhyZXNpZC5uZ3JhbSkgLSBhYnMocmVzaWQudmFuaWxsYSksCiAgICAgICAgIGJpZz1yZXNpZF9kaWZmIDwgLTEwKSAlPiUgCiAgZ2dwbG90KGFlcyh4PXN1cnByaXNhbC5uZ3JhbSwgY29sb3I9YmlnKSkgKyBnZW9tX2RlbnNpdHkoKSArIGZhY2V0X2dyaWQofmNvcnB1cykgKwogIGdndGl0bGUoIm5ncmFtIHN1cnByaXNhbCBvZiBoaWdoLWltcHJvdmVtZW50IHRva2VucyAocmVsYXRpdmUgdG8gdmFuaWxsYSkiKQoKcmVzaWRzX2pvaW5lZCAlPiUgCiAgbXV0YXRlKHJlc2lkX2Fic19kaWZmPWFicyhyZXNpZC5uZ3JhbSAtIHJlc2lkLnZhbmlsbGEpKSAlPiUgCiAgZ2dwbG90KGFlcyh4PWZyZXEubmdyYW0sIHk9cmVzaWRfYWJzX2RpZmYpKSArIGdlb21fcG9pbnQoYWxwaGE9MC4xKSArIGdlb21fc21vb3RoKCkKYGBgCgojIyMgSW52ZXN0aWdhdGUgZ3B0YnBlIHZzIHZhbmlsbGEKCmBgYHtyfQpncHRfcmVzaWRzID0gZnVsbF9yZXNpZHVhbHMgJT4lIGZpbHRlcihtb2RlbCA9PSAiZ3B0MiIsIHRyYWluaW5nID09ICJibGxpcC1zbS1ncHRicGUiKSAlPiUgZ3JvdXBfYnkoY29ycHVzLCBjb2RlKSAlPiUgc3VtbWFyaXNlKGZyZXE9bWVhbihmcmVxKSwgcHN5Y2hvbWV0cmljPW1lYW4ocHN5Y2hvbWV0cmljKSwgc3VycHJpc2FsPW1lYW4oc3VycHJpc2FsKSwgIHJlc2lkPW1lYW4ocmVzaWQpKQp2YW5pbGxhX3Jlc2lkcyA9IGZ1bGxfcmVzaWR1YWxzICU+JSBmaWx0ZXIobW9kZWwgPT0gInZhbmlsbGEiLCB0cmFpbmluZyA9PSAiYmxsaXAtc20iKSAlPiUgZ3JvdXBfYnkoY29ycHVzLCBjb2RlKSAlPiUgc3VtbWFyaXNlKGZyZXE9bWVhbihmcmVxKSwgcHN5Y2hvbWV0cmljPW1lYW4ocHN5Y2hvbWV0cmljKSwgc3VycHJpc2FsPW1lYW4oc3VycHJpc2FsKSwgIHJlc2lkPW1lYW4ocmVzaWQpKQpyZXNpZHNfam9pbmVkID0gZ3B0X3Jlc2lkcyAlPiUgbGVmdF9qb2luKHZhbmlsbGFfcmVzaWRzLCBieT1jKCJjb3JwdXMiLCAiY29kZSIpLCBzdWZmaXg9YygiLmdwdCIsICIudmFuaWxsYSIpKQoKcmVzaWRzX2pvaW5lZCAlPiUgCiAgZ2dwbG90KGFlcyh4PXJlc2lkLmdwdCwgeT1yZXNpZC52YW5pbGxhKSkgKyBnZW9tX3BvaW50KCkgKyBnZW9tX2FibGluZShzbG9wZT0xLCBjb2xvcj0icmVkIikgKwogIGZhY2V0X2dyaWQofmNvcnB1cykKCnJlc2lkc19qb2luZWQgJT4lIAogIG11dGF0ZShyZXNpZF9kaWZmPXJlc2lkLmdwdCAtIHJlc2lkLnZhbmlsbGEpICU+JSAKICBnZ3Bsb3QoYWVzKHg9cmVzaWRfZGlmZikpICsgZ2VvbV9kZW5zaXR5KCkgKwogIGZhY2V0X2dyaWQofmNvcnB1cykKCnJlc2lkc19qb2luZWQgJT4lIAogIG11dGF0ZShyZXNpZF9kaWZmPWFicyhyZXNpZC5ncHQpIC0gYWJzKHJlc2lkLnZhbmlsbGEpLAogICAgICAgICBiaWc9cmVzaWRfZGlmZiA8IC0xMCkgJT4lIAogIGdncGxvdChhZXMoeD1zdXJwcmlzYWwuZ3B0LCBjb2xvcj1iaWcpKSArIGdlb21fZGVuc2l0eSgpICsgZmFjZXRfZ3JpZCh+Y29ycHVzKSArCiAgZ2d0aXRsZSgiZ3B0IHN1cnByaXNhbCBvZiBoaWdoLWltcHJvdmVtZW50IHRva2VucyAocmVsYXRpdmUgdG8gdmFuaWxsYSkiKQoKcmVzaWRzX2pvaW5lZCAlPiUgCiAgbXV0YXRlKHJlc2lkX2Fic19kaWZmPWFicyhyZXNpZC5ncHQgLSByZXNpZC52YW5pbGxhKSkgJT4lIAogIGdncGxvdChhZXMoeD1mcmVxLmdwdCwgeT1yZXNpZF9hYnNfZGlmZikpICsgZ2VvbV9wb2ludChhbHBoYT0wLjEpICsgZ2VvbV9zbW9vdGgoKQpgYGAKCiMjIyBJbnZlc3RpZ2F0ZSByZXNpZHVhbHMgb3ZlcmFsbAoKYGBge3J9CnJlc2lkX2RlbHRhcyA9IGZ1bGxfcmVzaWR1YWxzICU+JSByaWdodF9qb2luKGJhc2VsaW5lX3Jlc2lkdWFscywgYnk9YygiY29ycHVzIiwgImNvZGUiLCAibW9kZWwiLCAidHJhaW5pbmciLCAic2VlZCIpLCBzdWZmaXg9YygiLmZ1bGwiLCAiLmJhc2VsaW5lIikpICU+JQogIHNlbGVjdChyZXNpZC5iYXNlbGluZSwgcmVzaWQuZnVsbCwgY29kZSwgc3VycHJpc2FsLmZ1bGwsIHBzeWNob21ldHJpYy5mdWxsLCBtb2RlbCwgdHJhaW5pbmcsIHNlZWQsIGNvcnB1cywgbGVuLmZ1bGwpICU+JQogIG11dGF0ZShyZXNpZC5iYXNlbGluZS5wb2wgPSBpZl9lbHNlKHJlc2lkLmJhc2VsaW5lID4gMCwgMSwgMCksCiAgICAgICAgIHJlc2lkLmZ1bGwucG9sID0gaWZfZWxzZShyZXNpZC5mdWxsID4gMCwgMSwgMCkpICU+JQogIG11dGF0ZShyZXNpZC5iYXNlbGluZSA9IGFicyhyZXNpZC5iYXNlbGluZSksCiAgICAgICAgIHJlc2lkLmZ1bGwgPSBhYnMocmVzaWQuZnVsbCkpICU+JQogIG11dGF0ZShyZXNpZF9kZWx0YT1yZXNpZC5iYXNlbGluZSAtIHJlc2lkLmZ1bGwsICNwb3NpdGl2ZSBpcyBiZXR0ZXIKICAgICAgICAgdHJhaW5pbmdfc291cmNlPWFzLmZhY3RvcihzdHJfcmVwbGFjZSh0cmFpbmluZywgIi1ncHRicGUiLCAiIikpLAogICAgICAgICBicGU9c3RyX2RldGVjdCh0cmFpbmluZywgImdwdGJwZSIpKQpgYGAKCmBgYHtyfQoKciA9IHJlc2lkX2RlbHRhcyAlPiUKICBmaWx0ZXIocmVzaWQuZnVsbC5wb2wgIT0gcmVzaWQuYmFzZWxpbmUucG9sKQoKYGBgCgpgYGB7cn0KcmVzaWRfZGVsdGFzICU+JQogIGdncGxvdChhZXMoeD1zdXJwcmlzYWwuZnVsbCwgeT1yZXNpZF9kZWx0YSwgY29sb3I9dHJhaW5pbmcpKSArCiAgICBmYWNldF9ncmlkKG1vZGVsfmNvcnB1cykgKwogICAgZ2VvbV9wb2ludChhbHBoYT0wLjEsIHNpemU9MC41KQpgYGAKCmBgYHtyfQpsYW5ndWFnZV9tb2RlbF9kYXRhICU+JSBmaWx0ZXIobW9kZWwgPT0gImdwdDIiKQpgYGAKCmBgYHtyfQpyZXNpZF9kZWx0YXMgJT4lCiAgZ3JvdXBfYnkoY29ycHVzKSAlPiUKICAgIG11dGF0ZShwc3ljaG9tZXRyaWMgPSBzY2FsZShwc3ljaG9tZXRyaWMuZnVsbCkpICU+JQogIHVuZ3JvdXAoKSAlPiUKICBnZ3Bsb3QoYWVzKHg9cHN5Y2hvbWV0cmljKSkgKwogICAgdGhlbWVfYncoKSArCiAgICBnZW9tX2RlbnNpdHkoKSArCiAgICBnZW9tX3ZsaW5lKHhpbnRlcmNlcHQgPSAwLCBjb2xvciA9ICJncmV5IikgKwogICAgZmFjZXRfZ3JpZCgufmNvcnB1cykgKwogICAgI2Nvb3JkX2NhcnRlc2lhbih4bGltID0gYygtMiwgNCkpICsKICAgIHRoZW1lKGF4aXMudGl0bGUueD1lbGVtZW50X2JsYW5rKCksCiAgICAgICAgICBheGlzLnRleHQueD1lbGVtZW50X2JsYW5rKCksCiAgICAgICAgICBheGlzLnRpY2tzLng9ZWxlbWVudF9ibGFuaygpKQojZ2dzYXZlKCJsZW5ndGgucG5nIiwgd2lkdGggPSA4LCBoZWlnaHQgPSAxKQoKYGBgCgpgYGB7cn0KCmxvZ19saWtfZGVsdGFzICAlPiUKI3Jlc2lkX2RlbHRhcyAlPiUKICAjZmlsdGVyKHJlc2lkLmZ1bGwucG9sID09IHJlc2lkLmJhc2VsaW5lLnBvbCkgJT4lCiAgZ3JvdXBfYnkoY29ycHVzKSAlPiUKICAgIG11dGF0ZShwc3ljaG9tZXRyaWMgPSBzY2FsZShwc3ljaG9tZXRyaWMpKSAlPiUKICB1bmdyb3VwKCkgJT4lCiAgI2ZpbHRlcihwc3ljaG9tZXRyaWMgPCA0KSAlPiUKICAjZmlsdGVyKGxlbi5mdWxsIDw9IDEwKSAlPiUKICBnZ3Bsb3QoYWVzKHggPSBwc3ljaG9tZXRyaWMsIHkgPSBkZWx0YV9sb2dfbGlrLCBjb2xvciA9IG1vZGVsKSkgKwogICAgdGhlbWVfYncoKSArCiAgICBmYWNldF9ncmlkKC4gfiBjb3JwdXMsIHNjYWxlcyA9ICJmcmVlIikgKwogICAgI2dlb21fcnVnKGFscGhhID0gMC4wMDMsIHNpZGVzID0gImIiKSArCiAgICBnZW9tX2hsaW5lKHlpbnRlcmNlcHQ9MCwgY29sb3IgPSAiYmx1ZSIpICsKICAgIGdlb21fdmxpbmUoeGludGVyY2VwdCA9IDAsIGNvbG9yID0gImdyZXkiKSArCiAgICBnZW9tX3Ntb290aChzZSA9IFQsIGFscGhhID0gMC4yKSArCiAgICBjb29yZF9jYXJ0ZXNpYW4oeWxpbSA9IGMoLTAuMSwgMC4yKSwgeGxpbSA9IGMoLTIsIDQpKSArCiAgICB0aGVtZShsZWdlbmQucG9zaXRpb24gPSAiYm90dG9tIiwKICAgICAgICAgIHN0cmlwLnRleHQueCA9IGVsZW1lbnRfYmxhbmsoKSkgCmdnc2F2ZSggIi4vcmVzaWRfcHN5Y2hvLnBuZyIsIGhlaWdodCA9IDQsIHdpZHRoID0gOCkKCgpgYGAKCmBgYHtyfQpyZXNpZF9kZWx0YXMgJT4lCiAgZ3JvdXBfYnkoY29ycHVzKSAlPiUKICAgIG11dGF0ZShwc3ljaG9tZXRyaWMgPSBzY2FsZShwc3ljaG9tZXRyaWMuZnVsbCkpICU+JQogIHVuZ3JvdXAoKSAlPiUKICBnZ3Bsb3QoYWVzKHg9bGVuLmZ1bGwpKSArCiAgICB0aGVtZV9idygpICsKICAgIGdlb21faGlzdG9ncmFtKGJpbnMgPSAyMCkgKwogICAgZ2VvbV92bGluZSh4aW50ZXJjZXB0ID0gMCwgY29sb3IgPSAiZ3JleSIpICsKICAgIGZhY2V0X2dyaWQoLn5jb3JwdXMpICsKICAgIGNvb3JkX2NhcnRlc2lhbih4bGltID0gYygxLCAxMCkpICsKICAgIHRoZW1lKGF4aXMudGl0bGUueD1lbGVtZW50X2JsYW5rKCksCiAgICAgICAgICBheGlzLnRleHQueD1lbGVtZW50X2JsYW5rKCksCiAgICAgICAgICBheGlzLnRpY2tzLng9ZWxlbWVudF9ibGFuaygpKQpnZ3NhdmUoImxlbmd0aF9tYXJpbmdhbHMucG5nIiwgd2lkdGggPSA4LCBoZWlnaHQgPSAxKQpgYGAKCmBgYHtyfQoKbG9nX2xpa19kZWx0YXMgICU+JQojcmVzaWRfZGVsdGFzICU+JQogICNmaWx0ZXIocmVzaWQuZnVsbC5wb2wgPT0gcmVzaWQuYmFzZWxpbmUucG9sKSAlPiUKICBncm91cF9ieShjb3JwdXMpICU+JQogICAgbXV0YXRlKHBzeWNob21ldHJpYyA9IHNjYWxlKHBzeWNob21ldHJpYykpICU+JQogIHVuZ3JvdXAoKSAlPiUKICAjZmlsdGVyKHBzeWNob21ldHJpYyA8IDQpICU+JQogICNmaWx0ZXIobGVuLmZ1bGwgPD0gMTApICU+JQogIGdncGxvdChhZXMoeCA9IGxlbiwgeSA9IGRlbHRhX2xvZ19saWssIGNvbG9yID0gbW9kZWwpKSArCiAgICB0aGVtZV9idygpICsKICAgIGZhY2V0X2dyaWQoLiB+IGNvcnB1cywgc2NhbGVzID0gImZyZWUiKSArCiAgICAjZ2VvbV9ydWcoYWxwaGEgPSAwLjAwMywgc2lkZXMgPSAiYiIpICsKICAgIGdlb21faGxpbmUoeWludGVyY2VwdD0wLCBjb2xvciA9ICJibHVlIikgKwogICAgZ2VvbV92bGluZSh4aW50ZXJjZXB0ID0gMCwgY29sb3IgPSAiZ3JleSIpICsKICAgIGdlb21fc21vb3RoKHNlID0gVCwgYWxwaGEgPSAwLjIpICsKICAgIGNvb3JkX2NhcnRlc2lhbih5bGltID0gYygtMC4wMiwgMC4wNiksIHhsaW0gPSBjKDEsIDEwKSkgKwogICAgdGhlbWUobGVnZW5kLnBvc2l0aW9uID0gImJvdHRvbSIsCiAgICAgICAgICBzdHJpcC50ZXh0LnggPSBlbGVtZW50X2JsYW5rKCkpIApnZ3NhdmUoICIuL3Jlc2lkX2xlbmd0aC5wbmciLCBoZWlnaHQgPSA0LCB3aWR0aCA9IDgpCmBgYAoKCmBgYHtyfQoKd29yZF9ub3JtID0gbG9nX2xpa19kZWx0YXMgJT4lCiAgZHJvcF9uYSgpICU+JQogIGdyb3VwX2J5KHdvcmQsIGNvcnB1cywgbW9kZWwsIHRyYWluaW5nLCBzZWVkKSAlPiUgCiAgbXV0YXRlKHBzeWNob3dvcmQgPSBzY2FsZShwc3ljaG9tZXRyaWMpLAogICAgICAgICBub3JtX3N1cnAgPSBzY2FsZShzdXJwcmlzYWwpKQpgYGAKCmBgYHtyfQp3b3JkX25vcm0gJT4lCiAgZ2dwbG90KGFlcyh4PW5vcm1fc3VycCkpICsKICBmYWNldF9ncmlkKH5jb3JwdXMpICsKICBnZW9tX2RlbnNpdHkoKSArCiAgY29vcmRfY2FydGVzaWFuKHhsaW0gPSBjKC0yLCA1KSkgKwogIGdlb21fdmxpbmUoeGludGVyY2VwdCA9IDAsIGNvbG9yID0gImdyZXkiKSArCiAgdGhlbWVfYncoKSArCiAgICB0aGVtZShheGlzLnRpdGxlLng9ZWxlbWVudF9ibGFuaygpLAogICAgICAgICAgYXhpcy50ZXh0Lng9ZWxlbWVudF9ibGFuaygpLAogICAgICAgICAgYXhpcy50aWNrcy54PWVsZW1lbnRfYmxhbmsoKSkKZ2dzYXZlKCJzdXJwX21hcmluZ2Fscy5wbmciLCB3aWR0aCA9IDgsIGhlaWdodCA9IDEpCgoKYGBgCmBgYHtyfQoKd29yZF9ub3JtICU+JQogIGdncGxvdChhZXMoeCA9IHBzeWNob3dvcmQsIHkgPSBub3JtX3N1cnAsIGNvbG9yID0gbW9kZWwpKSArCiAgICB0aGVtZV9idygpICsKICAgIGZhY2V0X2dyaWQoLiB+IGNvcnB1cywgc2NhbGVzID0gImZyZWUiKSArCiAgICAjZ2VvbV9ydWcoYWxwaGEgPSAwLjAwMywgc2lkZXMgPSAiYiIpICsKICAgIGdlb21faGxpbmUoeWludGVyY2VwdD0wLCBjb2xvciA9ICJibHVlIikgKwogICAgZ2VvbV92bGluZSh4aW50ZXJjZXB0ID0gMCwgY29sb3IgPSAiZ3JleSIpICsKICAgIGdlb21fc21vb3RoKHNlID0gVCwgYWxwaGEgPSAwLjIpICsKICAgICNjb29yZF9jYXJ0ZXNpYW4oeWxpbSA9IGMoLTAuMDUsIDAuMSksIHhsaW0gPSBjKC0yLCAzKSkgKwogICAgdGhlbWUobGVnZW5kLnBvc2l0aW9uID0gImJvdHRvbSIpIAojZ2dzYXZlKCAiLi9yZXNpZF9sZW5ndGgucG5nIiwgaGVpZ2h0ID0gNCwgd2lkdGggPSA4KQoKYGBgCgpgYGB7cn0KCndvcmRfbm9ybSAlPiUKICBnZ3Bsb3QoYWVzKHggPSBub3JtX3N1cnAsIHkgPSBkZWx0YV9sb2dfbGlrLCBjb2xvciA9IG1vZGVsKSkgKwogICAgdGhlbWVfYncoKSArCiAgICBmYWNldF9ncmlkKCAuIH4gY29ycHVzLCBzY2FsZXMgPSAiZnJlZSIpICsKICAgICNnZW9tX3J1ZyhhbHBoYSA9IDAuMDAzLCBzaWRlcyA9ICJiIikgKwogICAgZ2VvbV9obGluZSh5aW50ZXJjZXB0PTAsIGNvbG9yID0gImJsdWUiKSArCiAgICBnZW9tX3ZsaW5lKHhpbnRlcmNlcHQgPSAwLCBjb2xvciA9ICJncmV5IikgKwogICAgZ2VvbV9zbW9vdGgoc2UgPSBULCBhbHBoYSA9IDAuMikgKwogICAgY29vcmRfY2FydGVzaWFuKHlsaW0gPSBjKC0wLjA1LCAwLjA3KSwgeGxpbSA9IGMoLTIsIDUpKSArCiAgICB0aGVtZShsZWdlbmQucG9zaXRpb24gPSAiYm90dG9tIikgCmdnc2F2ZSggIi4vbm9ybV9zdXJwLnBuZyIsIGhlaWdodCA9IDQsIHdpZHRoID0gOCkKCmBgYAoKYGBge3J9CgpuZ3JhbV9oaWdoc3VycCA9IHdvcmRfbm9ybSAlPiUKICB1bmdyb3VwKCkgJT4lCiAgZmlsdGVyKGNvcnB1cyA9PSAiZHVuZGVlIiwgbm9ybV9zdXJwID4gMiwgbW9kZWwgPT0gIjVncmFtIikgJT4lCiAgc2VsZWN0KGNvZGUpCgpuZ3JhbV9oaWdoc3VycCA9IG5ncmFtX2hpZ2hzdXJwJGNvZGUKCnogPSB3b3JkX25vcm0gJT4lCiAgdW5ncm91cCgpICU+JQogIGZpbHRlcighIGNvZGUgJWluJSBuZ3JhbV9oaWdoc3VycCkgJT4lCiAgZmlsdGVyKGNvcnB1cyA9PSAiZHVuZGVlIikKCndyaXRlLmNzdih6LCAibmdyYW0tYWJsYXRlLmNzdiIpCgpgYGAKCgoK